VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 93705

Last change on this file since 93705 was 93574, checked in by vboxsync, 3 years ago

VMM/HM: Nested VMX: bugref:10092 Separate out VM-exit counters for each guest run-loop (normal, debug, nested).
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 380.1 KB
Line 
1/* $Id: HMSVMR0.cpp 93574 2022-02-03 11:27:27Z vboxsync $ */
2/** @file
3 * HM SVM (AMD-V) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2013-2022 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/tm.h>
32#include <VBox/vmm/em.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#include "HMInternal.h"
36#include <VBox/vmm/vmcc.h>
37#include <VBox/err.h>
38#include "HMSVMR0.h"
39#include "dtrace/VBoxVMM.h"
40
41#ifdef DEBUG_ramshankar
42# define HMSVM_SYNC_FULL_GUEST_STATE
43# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
44# define HMSVM_ALWAYS_TRAP_PF
45# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
46#endif
47
48
49/*********************************************************************************************************************************
50* Defined Constants And Macros *
51*********************************************************************************************************************************/
52#ifdef VBOX_WITH_STATISTICS
53# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
54 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
55 if ((u64ExitCode) == SVM_EXIT_NPF) \
56 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
57 else \
58 STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
59 } while (0)
60
61# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
62 STAM_COUNTER_INC(&pVCpu->hm.s.StatDebugExitAll); \
63 if ((u64ExitCode) == SVM_EXIT_NPF) \
64 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
65 else \
66 STAM_COUNTER_INC(&pVCpu->hm.s.aStatExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
67 } while (0)
68
69# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
70 STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitAll); \
71 if ((u64ExitCode) == SVM_EXIT_NPF) \
72 STAM_COUNTER_INC(&pVCpu->hm.s.StatNestedExitReasonNpf); \
73 else \
74 STAM_COUNTER_INC(&pVCpu->hm.s.aStatNestedExitReason[(u64ExitCode) & MASK_EXITREASON_STAT]); \
75 } while (0)
76#else
77# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
78# define HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
79# define HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
80#endif /* !VBOX_WITH_STATISTICS */
81
82/** If we decide to use a function table approach this can be useful to
83 * switch to a "static DECLCALLBACK(int)". */
84#define HMSVM_EXIT_DECL static VBOXSTRICTRC
85
86/**
87 * Subset of the guest-CPU state that is kept by SVM R0 code while executing the
88 * guest using hardware-assisted SVM.
89 *
90 * This excludes state like TSC AUX, GPRs (other than RSP, RAX) which are always
91 * are swapped and restored across the world-switch and also registers like
92 * EFER, PAT MSR etc. which cannot be modified by the guest without causing a
93 * \#VMEXIT.
94 */
95#define HMSVM_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
96 | CPUMCTX_EXTRN_RFLAGS \
97 | CPUMCTX_EXTRN_RAX \
98 | CPUMCTX_EXTRN_RSP \
99 | CPUMCTX_EXTRN_SREG_MASK \
100 | CPUMCTX_EXTRN_CR0 \
101 | CPUMCTX_EXTRN_CR2 \
102 | CPUMCTX_EXTRN_CR3 \
103 | CPUMCTX_EXTRN_TABLE_MASK \
104 | CPUMCTX_EXTRN_DR6 \
105 | CPUMCTX_EXTRN_DR7 \
106 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
107 | CPUMCTX_EXTRN_SYSCALL_MSRS \
108 | CPUMCTX_EXTRN_SYSENTER_MSRS \
109 | CPUMCTX_EXTRN_HWVIRT \
110 | CPUMCTX_EXTRN_INHIBIT_INT \
111 | CPUMCTX_EXTRN_HM_SVM_MASK)
112
113/**
114 * Subset of the guest-CPU state that is shared between the guest and host.
115 */
116#define HMSVM_CPUMCTX_SHARED_STATE CPUMCTX_EXTRN_DR_MASK
117
118/** Macro for importing guest state from the VMCB back into CPUMCTX. */
119#define HMSVM_CPUMCTX_IMPORT_STATE(a_pVCpu, a_fWhat) \
120 do { \
121 if ((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fWhat)) \
122 hmR0SvmImportGuestState((a_pVCpu), (a_fWhat)); \
123 } while (0)
124
125/** Assert that the required state bits are fetched. */
126#define HMSVM_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
127 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
128 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
129
130/** Assert that preemption is disabled or covered by thread-context hooks. */
131#define HMSVM_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
132 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
133
134/** Assert that we haven't migrated CPUs when thread-context hooks are not
135 * used. */
136#define HMSVM_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
137 || (a_pVCpu)->hmr0.s.idEnteredCpu == RTMpCpuId(), \
138 ("Illegal migration! Entered on CPU %u Current %u\n", \
139 (a_pVCpu)->hmr0.s.idEnteredCpu, RTMpCpuId()));
140
141/** Assert that we're not executing a nested-guest. */
142#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
143# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
144#else
145# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
146#endif
147
148/** Assert that we're executing a nested-guest. */
149#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
150# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
151#else
152# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
153#endif
154
155/** Macro for checking and returning from the using function for
156 * \#VMEXIT intercepts that maybe caused during delivering of another
157 * event in the guest. */
158#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
159# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
160 do \
161 { \
162 int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
163 if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
164 else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
165 else if ( rc == VINF_EM_RESET \
166 && CPUMIsGuestSvmCtrlInterceptSet((a_pVCpu), &(a_pVCpu)->cpum.GstCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
167 { \
168 HMSVM_CPUMCTX_IMPORT_STATE((a_pVCpu), HMSVM_CPUMCTX_EXTRN_ALL); \
169 return IEMExecSvmVmexit((a_pVCpu), SVM_EXIT_SHUTDOWN, 0, 0); \
170 } \
171 else \
172 return rc; \
173 } while (0)
174#else
175# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(a_pVCpu, a_pSvmTransient) \
176 do \
177 { \
178 int rc = hmR0SvmCheckExitDueToEventDelivery((a_pVCpu), (a_pSvmTransient)); \
179 if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
180 else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
181 else \
182 return rc; \
183 } while (0)
184#endif
185
186/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
187 * instruction that exited. */
188#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
189 do { \
190 if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
191 (a_rc) = VINF_EM_DBG_STEPPED; \
192 } while (0)
193
194/** Validate segment descriptor granularity bit. */
195#ifdef VBOX_STRICT
196# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) \
197 AssertMsg( !(a_pCtx)->reg.Attr.n.u1Present \
198 || ( (a_pCtx)->reg.Attr.n.u1Granularity \
199 ? ((a_pCtx)->reg.u32Limit & 0xfff) == 0xfff \
200 : (a_pCtx)->reg.u32Limit <= UINT32_C(0xfffff)), \
201 ("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", (a_pCtx)->reg.u32Limit, \
202 (a_pCtx)->reg.Attr.u, (a_pCtx)->reg.u64Base))
203#else
204# define HMSVM_ASSERT_SEG_GRANULARITY(a_pCtx, reg) do { } while (0)
205#endif
206
207/**
208 * Exception bitmap mask for all contributory exceptions.
209 *
210 * Page fault is deliberately excluded here as it's conditional as to whether
211 * it's contributory or benign. Page faults are handled separately.
212 */
213#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
214 | RT_BIT(X86_XCPT_DE))
215
216/**
217 * Mandatory/unconditional guest control intercepts.
218 *
219 * SMIs can and do happen in normal operation. We need not intercept them
220 * while executing the guest (or nested-guest).
221 */
222#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
223 | SVM_CTRL_INTERCEPT_NMI \
224 | SVM_CTRL_INTERCEPT_INIT \
225 | SVM_CTRL_INTERCEPT_RDPMC \
226 | SVM_CTRL_INTERCEPT_CPUID \
227 | SVM_CTRL_INTERCEPT_RSM \
228 | SVM_CTRL_INTERCEPT_HLT \
229 | SVM_CTRL_INTERCEPT_IOIO_PROT \
230 | SVM_CTRL_INTERCEPT_MSR_PROT \
231 | SVM_CTRL_INTERCEPT_INVLPGA \
232 | SVM_CTRL_INTERCEPT_SHUTDOWN \
233 | SVM_CTRL_INTERCEPT_FERR_FREEZE \
234 | SVM_CTRL_INTERCEPT_VMRUN \
235 | SVM_CTRL_INTERCEPT_SKINIT \
236 | SVM_CTRL_INTERCEPT_WBINVD \
237 | SVM_CTRL_INTERCEPT_MONITOR \
238 | SVM_CTRL_INTERCEPT_MWAIT \
239 | SVM_CTRL_INTERCEPT_CR0_SEL_WRITE \
240 | SVM_CTRL_INTERCEPT_XSETBV)
241
242/** @name VMCB Clean Bits.
243 *
244 * These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
245 * AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
246 * memory.
247 *
248 * @{ */
249/** All intercepts vectors, TSC offset, PAUSE filter counter. */
250#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
251/** I/O permission bitmap, MSR permission bitmap. */
252#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
253/** ASID. */
254#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
255/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
256V_INTR_VECTOR. */
257#define HMSVM_VMCB_CLEAN_INT_CTRL RT_BIT(3)
258/** Nested Paging: Nested CR3 (nCR3), PAT. */
259#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
260/** Control registers (CR0, CR3, CR4, EFER). */
261#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
262/** Debug registers (DR6, DR7). */
263#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
264/** GDT, IDT limit and base. */
265#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
266/** Segment register: CS, SS, DS, ES limit and base. */
267#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
268/** CR2.*/
269#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
270/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
271#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
272/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
273PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
274#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
275/** Mask of all valid VMCB Clean bits. */
276#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
277 | HMSVM_VMCB_CLEAN_IOPM_MSRPM \
278 | HMSVM_VMCB_CLEAN_ASID \
279 | HMSVM_VMCB_CLEAN_INT_CTRL \
280 | HMSVM_VMCB_CLEAN_NP \
281 | HMSVM_VMCB_CLEAN_CRX_EFER \
282 | HMSVM_VMCB_CLEAN_DRX \
283 | HMSVM_VMCB_CLEAN_DT \
284 | HMSVM_VMCB_CLEAN_SEG \
285 | HMSVM_VMCB_CLEAN_CR2 \
286 | HMSVM_VMCB_CLEAN_LBR \
287 | HMSVM_VMCB_CLEAN_AVIC)
288/** @} */
289
290/** @name SVM transient.
291 *
292 * A state structure for holding miscellaneous information across AMD-V
293 * VMRUN/\#VMEXIT operation, restored after the transition.
294 *
295 * @{ */
296typedef struct SVMTRANSIENT
297{
298 /** The host's rflags/eflags. */
299 RTCCUINTREG fEFlags;
300 /** The \#VMEXIT exit code (the EXITCODE field in the VMCB). */
301 uint64_t u64ExitCode;
302
303 /** The guest's TPR value used for TPR shadowing. */
304 uint8_t u8GuestTpr;
305 /** Alignment. */
306 uint8_t abAlignment0[7];
307
308 /** Pointer to the currently executing VMCB. */
309 PSVMVMCB pVmcb;
310
311 /** Whether we are currently executing a nested-guest. */
312 bool fIsNestedGuest;
313 /** Whether the guest debug state was active at the time of \#VMEXIT. */
314 bool fWasGuestDebugStateActive;
315 /** Whether the hyper debug state was active at the time of \#VMEXIT. */
316 bool fWasHyperDebugStateActive;
317 /** Whether the TSC offset mode needs to be updated. */
318 bool fUpdateTscOffsetting;
319 /** Whether the TSC_AUX MSR needs restoring on \#VMEXIT. */
320 bool fRestoreTscAuxMsr;
321 /** Whether the \#VMEXIT was caused by a page-fault during delivery of a
322 * contributary exception or a page-fault. */
323 bool fVectoringDoublePF;
324 /** Whether the \#VMEXIT was caused by a page-fault during delivery of an
325 * external interrupt or NMI. */
326 bool fVectoringPF;
327 /** Padding. */
328 bool afPadding0;
329} SVMTRANSIENT;
330/** Pointer to SVM transient state. */
331typedef SVMTRANSIENT *PSVMTRANSIENT;
332/** Pointer to a const SVM transient state. */
333typedef const SVMTRANSIENT *PCSVMTRANSIENT;
334
335AssertCompileSizeAlignment(SVMTRANSIENT, sizeof(uint64_t));
336AssertCompileMemberAlignment(SVMTRANSIENT, u64ExitCode, sizeof(uint64_t));
337AssertCompileMemberAlignment(SVMTRANSIENT, pVmcb, sizeof(uint64_t));
338/** @} */
339
340/**
341 * MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
342 */
343typedef enum SVMMSREXITREAD
344{
345 /** Reading this MSR causes a \#VMEXIT. */
346 SVMMSREXIT_INTERCEPT_READ = 0xb,
347 /** Reading this MSR does not cause a \#VMEXIT. */
348 SVMMSREXIT_PASSTHRU_READ
349} SVMMSREXITREAD;
350
351/**
352 * MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
353 */
354typedef enum SVMMSREXITWRITE
355{
356 /** Writing to this MSR causes a \#VMEXIT. */
357 SVMMSREXIT_INTERCEPT_WRITE = 0xd,
358 /** Writing to this MSR does not cause a \#VMEXIT. */
359 SVMMSREXIT_PASSTHRU_WRITE
360} SVMMSREXITWRITE;
361
362/**
363 * SVM \#VMEXIT handler.
364 *
365 * @returns Strict VBox status code.
366 * @param pVCpu The cross context virtual CPU structure.
367 * @param pSvmTransient Pointer to the SVM-transient structure.
368 */
369typedef VBOXSTRICTRC FNSVMEXITHANDLER(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
370
371
372/*********************************************************************************************************************************
373* Internal Functions *
374*********************************************************************************************************************************/
375static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu);
376static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState);
377
378
379/** @name \#VMEXIT handlers.
380 * @{
381 */
382static FNSVMEXITHANDLER hmR0SvmExitIntr;
383static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
384static FNSVMEXITHANDLER hmR0SvmExitInvd;
385static FNSVMEXITHANDLER hmR0SvmExitCpuid;
386static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
387static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
388static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
389static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
390static FNSVMEXITHANDLER hmR0SvmExitHlt;
391static FNSVMEXITHANDLER hmR0SvmExitMonitor;
392static FNSVMEXITHANDLER hmR0SvmExitMwait;
393static FNSVMEXITHANDLER hmR0SvmExitShutdown;
394static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
395static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
396static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
397static FNSVMEXITHANDLER hmR0SvmExitMsr;
398static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
399static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
400static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
401static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
402static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
403static FNSVMEXITHANDLER hmR0SvmExitVIntr;
404static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
405static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
406static FNSVMEXITHANDLER hmR0SvmExitPause;
407static FNSVMEXITHANDLER hmR0SvmExitFerrFreeze;
408static FNSVMEXITHANDLER hmR0SvmExitIret;
409static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
410static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
411static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
412static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
413static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
414static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
415static FNSVMEXITHANDLER hmR0SvmExitXcptGP;
416static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
417static FNSVMEXITHANDLER hmR0SvmExitSwInt;
418static FNSVMEXITHANDLER hmR0SvmExitTrRead;
419static FNSVMEXITHANDLER hmR0SvmExitTrWrite;
420#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
421static FNSVMEXITHANDLER hmR0SvmExitClgi;
422static FNSVMEXITHANDLER hmR0SvmExitStgi;
423static FNSVMEXITHANDLER hmR0SvmExitVmload;
424static FNSVMEXITHANDLER hmR0SvmExitVmsave;
425static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
426static FNSVMEXITHANDLER hmR0SvmExitVmrun;
427static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
428static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
429#endif
430/** @} */
431
432static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
433#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
434static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient);
435#endif
436static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops);
437
438
439/*********************************************************************************************************************************
440* Global Variables *
441*********************************************************************************************************************************/
442/** Ring-0 memory object for the IO bitmap. */
443static RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
444/** Physical address of the IO bitmap. */
445static RTHCPHYS g_HCPhysIOBitmap;
446/** Pointer to the IO bitmap. */
447static R0PTRTYPE(void *) g_pvIOBitmap;
448
449#ifdef VBOX_STRICT
450# define HMSVM_LOG_RBP_RSP RT_BIT_32(0)
451# define HMSVM_LOG_CR_REGS RT_BIT_32(1)
452# define HMSVM_LOG_CS RT_BIT_32(2)
453# define HMSVM_LOG_SS RT_BIT_32(3)
454# define HMSVM_LOG_FS RT_BIT_32(4)
455# define HMSVM_LOG_GS RT_BIT_32(5)
456# define HMSVM_LOG_LBR RT_BIT_32(6)
457# define HMSVM_LOG_ALL ( HMSVM_LOG_RBP_RSP \
458 | HMSVM_LOG_CR_REGS \
459 | HMSVM_LOG_CS \
460 | HMSVM_LOG_SS \
461 | HMSVM_LOG_FS \
462 | HMSVM_LOG_GS \
463 | HMSVM_LOG_LBR)
464
465/**
466 * Dumps virtual CPU state and additional info. to the logger for diagnostics.
467 *
468 * @param pVCpu The cross context virtual CPU structure.
469 * @param pVmcb Pointer to the VM control block.
470 * @param pszPrefix Log prefix.
471 * @param fFlags Log flags, see HMSVM_LOG_XXX.
472 * @param uVerbose The verbosity level, currently unused.
473 */
474static void hmR0SvmLogState(PVMCPUCC pVCpu, PCSVMVMCB pVmcb, const char *pszPrefix, uint32_t fFlags, uint8_t uVerbose)
475{
476 RT_NOREF2(pVCpu, uVerbose);
477 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
478
479 HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
480 Log4(("%s: cs:rip=%04x:%RX64 efl=%#RX64\n", pszPrefix, pCtx->cs.Sel, pCtx->rip, pCtx->rflags.u));
481
482 if (fFlags & HMSVM_LOG_RBP_RSP)
483 {
484 HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RBP);
485 Log4(("%s: rsp=%#RX64 rbp=%#RX64\n", pszPrefix, pCtx->rsp, pCtx->rbp));
486 }
487
488 if (fFlags & HMSVM_LOG_CR_REGS)
489 {
490 HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4);
491 Log4(("%s: cr0=%#RX64 cr3=%#RX64 cr4=%#RX64\n", pszPrefix, pCtx->cr0, pCtx->cr3, pCtx->cr4));
492 }
493
494 if (fFlags & HMSVM_LOG_CS)
495 {
496 HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
497 Log4(("%s: cs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->cs.Sel, pCtx->cs.u64Base,
498 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
499 }
500 if (fFlags & HMSVM_LOG_SS)
501 {
502 HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
503 Log4(("%s: ss={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->ss.Sel, pCtx->ss.u64Base,
504 pCtx->ss.u32Limit, pCtx->ss.Attr.u));
505 }
506 if (fFlags & HMSVM_LOG_FS)
507 {
508 HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
509 Log4(("%s: fs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->fs.Sel, pCtx->fs.u64Base,
510 pCtx->fs.u32Limit, pCtx->fs.Attr.u));
511 }
512 if (fFlags & HMSVM_LOG_GS)
513 {
514 HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
515 Log4(("%s: gs={%04x base=%016RX64 limit=%08x flags=%08x}\n", pszPrefix, pCtx->gs.Sel, pCtx->gs.u64Base,
516 pCtx->gs.u32Limit, pCtx->gs.Attr.u));
517 }
518
519 PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
520 if (fFlags & HMSVM_LOG_LBR)
521 {
522 Log4(("%s: br_from=%#RX64 br_to=%#RX64 lastxcpt_from=%#RX64 lastxcpt_to=%#RX64\n", pszPrefix, pVmcbGuest->u64BR_FROM,
523 pVmcbGuest->u64BR_TO, pVmcbGuest->u64LASTEXCPFROM, pVmcbGuest->u64LASTEXCPTO));
524 }
525 NOREF(pszPrefix); NOREF(pVmcbGuest); NOREF(pCtx);
526}
527#endif /* VBOX_STRICT */
528
529
530/**
531 * Sets up and activates AMD-V on the current CPU.
532 *
533 * @returns VBox status code.
534 * @param pHostCpu The HM physical-CPU structure.
535 * @param pVM The cross context VM structure. Can be
536 * NULL after a resume!
537 * @param pvCpuPage Pointer to the global CPU page.
538 * @param HCPhysCpuPage Physical address of the global CPU page.
539 * @param fEnabledByHost Whether the host OS has already initialized AMD-V.
540 * @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs (currently
541 * unused).
542 */
543VMMR0DECL(int) SVMR0EnableCpu(PHMPHYSCPU pHostCpu, PVMCC pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
544 PCSUPHWVIRTMSRS pHwvirtMsrs)
545{
546 Assert(!fEnabledByHost);
547 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
548 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
549 Assert(pvCpuPage); NOREF(pvCpuPage);
550 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
551
552 RT_NOREF2(fEnabledByHost, pHwvirtMsrs);
553
554 /* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
555 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
556
557 /*
558 * We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
559 */
560 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
561 if (u64HostEfer & MSR_K6_EFER_SVME)
562 {
563 /* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
564 if ( pVM
565 && pVM->hm.s.svm.fIgnoreInUseError)
566 pHostCpu->fIgnoreAMDVInUseError = true;
567
568 if (!pHostCpu->fIgnoreAMDVInUseError)
569 {
570 ASMSetFlags(fEFlags);
571 return VERR_SVM_IN_USE;
572 }
573 }
574
575 /* Turn on AMD-V in the EFER MSR. */
576 ASMWrMsr(MSR_K6_EFER, u64HostEfer | MSR_K6_EFER_SVME);
577
578 /* Write the physical page address where the CPU will store the host state while executing the VM. */
579 ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
580
581 /* Restore interrupts. */
582 ASMSetFlags(fEFlags);
583
584 /*
585 * Theoretically, other hypervisors may have used ASIDs, ideally we should flush all
586 * non-zero ASIDs when enabling SVM. AMD doesn't have an SVM instruction to flush all
587 * ASIDs (flushing is done upon VMRUN). Therefore, flag that we need to flush the TLB
588 * entirely with before executing any guest code.
589 */
590 pHostCpu->fFlushAsidBeforeUse = true;
591
592 /*
593 * Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
594 */
595 ++pHostCpu->cTlbFlushes;
596
597 return VINF_SUCCESS;
598}
599
600
601/**
602 * Deactivates AMD-V on the current CPU.
603 *
604 * @returns VBox status code.
605 * @param pHostCpu The HM physical-CPU structure.
606 * @param pvCpuPage Pointer to the global CPU page.
607 * @param HCPhysCpuPage Physical address of the global CPU page.
608 */
609VMMR0DECL(int) SVMR0DisableCpu(PHMPHYSCPU pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
610{
611 RT_NOREF1(pHostCpu);
612 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
613 AssertReturn( HCPhysCpuPage
614 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
615 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
616
617 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
618 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
619
620 /* Turn off AMD-V in the EFER MSR. */
621 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
622 ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
623
624 /* Invalidate host state physical address. */
625 ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
626
627 /* Restore interrupts. */
628 ASMSetFlags(fEFlags);
629
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Does global AMD-V initialization (called during module initialization).
636 *
637 * @returns VBox status code.
638 */
639VMMR0DECL(int) SVMR0GlobalInit(void)
640{
641 /*
642 * Allocate 12 KB (3 pages) for the IO bitmap. Since this is non-optional and we always
643 * intercept all IO accesses, it's done once globally here instead of per-VM.
644 */
645 Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
646 int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
647 if (RT_FAILURE(rc))
648 return rc;
649
650 g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
651 g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
652
653 /* Set all bits to intercept all IO accesses. */
654 ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
655
656 return VINF_SUCCESS;
657}
658
659
660/**
661 * Does global AMD-V termination (called during module termination).
662 */
663VMMR0DECL(void) SVMR0GlobalTerm(void)
664{
665 if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
666 {
667 RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
668 g_pvIOBitmap = NULL;
669 g_HCPhysIOBitmap = 0;
670 g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
671 }
672}
673
674
675/**
676 * Frees any allocated per-VCPU structures for a VM.
677 *
678 * @param pVM The cross context VM structure.
679 */
680DECLINLINE(void) hmR0SvmFreeStructs(PVMCC pVM)
681{
682 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
683 {
684 PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
685 AssertPtr(pVCpu);
686
687 if (pVCpu->hmr0.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
688 {
689 RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcbHost, false);
690 pVCpu->hmr0.s.svm.HCPhysVmcbHost = 0;
691 pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
692 }
693
694 if (pVCpu->hmr0.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
695 {
696 RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjVmcb, false);
697 pVCpu->hmr0.s.svm.pVmcb = NULL;
698 pVCpu->hmr0.s.svm.HCPhysVmcb = 0;
699 pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
700 }
701
702 if (pVCpu->hmr0.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
703 {
704 RTR0MemObjFree(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, false);
705 pVCpu->hmr0.s.svm.pvMsrBitmap = NULL;
706 pVCpu->hmr0.s.svm.HCPhysMsrBitmap = 0;
707 pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
708 }
709 }
710}
711
712
713/**
714 * Sets pfnVMRun to the best suited variant.
715 *
716 * This must be called whenever anything changes relative to the SVMR0VMRun
717 * variant selection:
718 * - pVCpu->hm.s.fLoadSaveGuestXcr0
719 * - CPUMCTX_WSF_IBPB_ENTRY in pVCpu->cpum.GstCtx.fWorldSwitcher
720 * - CPUMCTX_WSF_IBPB_EXIT in pVCpu->cpum.GstCtx.fWorldSwitcher
721 * - Perhaps: CPUMIsGuestFPUStateActive() (windows only)
722 * - Perhaps: CPUMCTX.fXStateMask (windows only)
723 *
724 * We currently ASSUME that neither CPUMCTX_WSF_IBPB_ENTRY nor
725 * CPUMCTX_WSF_IBPB_EXIT cannot be changed at runtime.
726 */
727static void hmR0SvmUpdateVmRunFunction(PVMCPUCC pVCpu)
728{
729 static const struct CLANGWORKAROUND { PFNHMSVMVMRUN pfn; } s_aHmR0SvmVmRunFunctions[] =
730 {
731 { hmR0SvmVmRun_SansXcr0_SansIbpbEntry_SansIbpbExit },
732 { hmR0SvmVmRun_WithXcr0_SansIbpbEntry_SansIbpbExit },
733 { hmR0SvmVmRun_SansXcr0_WithIbpbEntry_SansIbpbExit },
734 { hmR0SvmVmRun_WithXcr0_WithIbpbEntry_SansIbpbExit },
735 { hmR0SvmVmRun_SansXcr0_SansIbpbEntry_WithIbpbExit },
736 { hmR0SvmVmRun_WithXcr0_SansIbpbEntry_WithIbpbExit },
737 { hmR0SvmVmRun_SansXcr0_WithIbpbEntry_WithIbpbExit },
738 { hmR0SvmVmRun_WithXcr0_WithIbpbEntry_WithIbpbExit },
739 };
740 uintptr_t const idx = (pVCpu->hmr0.s.fLoadSaveGuestXcr0 ? 1 : 0)
741 | (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_ENTRY ? 2 : 0)
742 | (pVCpu->hmr0.s.fWorldSwitcher & HM_WSF_IBPB_EXIT ? 4 : 0);
743 PFNHMSVMVMRUN const pfnVMRun = s_aHmR0SvmVmRunFunctions[idx].pfn;
744 if (pVCpu->hmr0.s.svm.pfnVMRun != pfnVMRun)
745 pVCpu->hmr0.s.svm.pfnVMRun = pfnVMRun;
746}
747
748
749/**
750 * Selector FNHMSVMVMRUN implementation.
751 */
752static DECLCALLBACK(int) hmR0SvmVMRunSelector(PVMCC pVM, PVMCPUCC pVCpu, RTHCPHYS HCPhysVMCB)
753{
754 hmR0SvmUpdateVmRunFunction(pVCpu);
755 return pVCpu->hmr0.s.svm.pfnVMRun(pVM, pVCpu, HCPhysVMCB);
756}
757
758
759/**
760 * Does per-VM AMD-V initialization.
761 *
762 * @returns VBox status code.
763 * @param pVM The cross context VM structure.
764 */
765VMMR0DECL(int) SVMR0InitVM(PVMCC pVM)
766{
767 int rc = VERR_INTERNAL_ERROR_5;
768
769 /*
770 * Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
771 */
772 uint32_t u32Family;
773 uint32_t u32Model;
774 uint32_t u32Stepping;
775 if (HMIsSubjectToSvmErratum170(&u32Family, &u32Model, &u32Stepping))
776 {
777 Log4Func(("AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
778 pVM->hmr0.s.svm.fAlwaysFlushTLB = true;
779 }
780
781 /*
782 * Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
783 */
784 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
785 {
786 PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
787 pVCpu->hmr0.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
788 pVCpu->hmr0.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
789 pVCpu->hmr0.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
790 }
791
792 for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
793 {
794 PVMCPUCC pVCpu = VMCC_GET_CPU(pVM, idCpu);
795
796 /*
797 * Initialize the hardware-assisted SVM guest-execution handler.
798 * We now use a single handler for both 32-bit and 64-bit guests, see @bugref{6208#c73}.
799 */
800 pVCpu->hmr0.s.svm.pfnVMRun = hmR0SvmVMRunSelector;
801
802 /*
803 * Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
804 * FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
805 */
806/** @todo Does this need to be below 4G? */
807 rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << HOST_PAGE_SHIFT, false /* fExecutable */);
808 if (RT_FAILURE(rc))
809 goto failure_cleanup;
810
811 void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcbHost);
812 pVCpu->hmr0.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcbHost, 0 /* iPage */);
813 Assert(pVCpu->hmr0.s.svm.HCPhysVmcbHost < _4G);
814 RT_BZERO(pvVmcbHost, HOST_PAGE_SIZE);
815
816 /*
817 * Allocate one page for the guest-state VMCB.
818 */
819/** @todo Does this need to be below 4G? */
820 rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << HOST_PAGE_SHIFT, false /* fExecutable */);
821 if (RT_FAILURE(rc))
822 goto failure_cleanup;
823
824 pVCpu->hmr0.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjVmcb);
825 pVCpu->hmr0.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjVmcb, 0 /* iPage */);
826 Assert(pVCpu->hmr0.s.svm.HCPhysVmcb < _4G);
827 RT_BZERO(pVCpu->hmr0.s.svm.pVmcb, HOST_PAGE_SIZE);
828
829 /*
830 * Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
831 * SVM to not require one.
832 */
833/** @todo Does this need to be below 4G? */
834 rc = RTR0MemObjAllocCont(&pVCpu->hmr0.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT,
835 false /* fExecutable */);
836 if (RT_FAILURE(rc))
837 goto failure_cleanup;
838
839 pVCpu->hmr0.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hmr0.s.svm.hMemObjMsrBitmap);
840 pVCpu->hmr0.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hmr0.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
841 /* Set all bits to intercept all MSR accesses (changed later on). */
842 ASMMemFill32(pVCpu->hmr0.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << HOST_PAGE_SHIFT, UINT32_C(0xffffffff));
843 }
844
845 return VINF_SUCCESS;
846
847failure_cleanup:
848 hmR0SvmFreeStructs(pVM);
849 return rc;
850}
851
852
853/**
854 * Does per-VM AMD-V termination.
855 *
856 * @returns VBox status code.
857 * @param pVM The cross context VM structure.
858 */
859VMMR0DECL(int) SVMR0TermVM(PVMCC pVM)
860{
861 hmR0SvmFreeStructs(pVM);
862 return VINF_SUCCESS;
863}
864
865
866/**
867 * Returns whether the VMCB Clean Bits feature is supported.
868 *
869 * @returns @c true if supported, @c false otherwise.
870 * @param pVCpu The cross context virtual CPU structure.
871 * @param fIsNestedGuest Whether we are currently executing the nested-guest.
872 */
873DECL_FORCE_INLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPUCC pVCpu, bool fIsNestedGuest)
874{
875 PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
876 bool const fHostVmcbCleanBits = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
877 if (!fIsNestedGuest)
878 return fHostVmcbCleanBits;
879 return fHostVmcbCleanBits && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
880}
881
882
883/**
884 * Returns whether the decode assists feature is supported.
885 *
886 * @returns @c true if supported, @c false otherwise.
887 * @param pVCpu The cross context virtual CPU structure.
888 */
889DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPUCC pVCpu)
890{
891 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
892#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
893 if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
894 return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
895 && pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
896#endif
897 return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
898}
899
900
901/**
902 * Returns whether the NRIP_SAVE feature is supported.
903 *
904 * @returns @c true if supported, @c false otherwise.
905 * @param pVCpu The cross context virtual CPU structure.
906 */
907DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPUCC pVCpu)
908{
909 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
910#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
911 if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
912 return (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
913 && pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
914#endif
915 return RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
916}
917
918
919/**
920 * Sets the permission bits for the specified MSR in the MSRPM bitmap.
921 *
922 * @param pVCpu The cross context virtual CPU structure.
923 * @param pbMsrBitmap Pointer to the MSR bitmap.
924 * @param idMsr The MSR for which the permissions are being set.
925 * @param enmRead MSR read permissions.
926 * @param enmWrite MSR write permissions.
927 *
928 * @remarks This function does -not- clear the VMCB clean bits for MSRPM. The
929 * caller needs to take care of this.
930 */
931static void hmR0SvmSetMsrPermission(PVMCPUCC pVCpu, uint8_t *pbMsrBitmap, uint32_t idMsr, SVMMSREXITREAD enmRead,
932 SVMMSREXITWRITE enmWrite)
933{
934 bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx);
935 uint16_t offMsrpm;
936 uint8_t uMsrpmBit;
937 int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
938 AssertRC(rc);
939
940 Assert(uMsrpmBit == 0 || uMsrpmBit == 2 || uMsrpmBit == 4 || uMsrpmBit == 6);
941 Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
942
943 pbMsrBitmap += offMsrpm;
944 if (enmRead == SVMMSREXIT_INTERCEPT_READ)
945 *pbMsrBitmap |= RT_BIT(uMsrpmBit);
946 else
947 {
948 if (!fInNestedGuestMode)
949 *pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
950#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
951 else
952 {
953 /* Only clear the bit if the nested-guest is also not intercepting the MSR read.*/
954 if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit)))
955 *pbMsrBitmap &= ~RT_BIT(uMsrpmBit);
956 else
957 Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit));
958 }
959#endif
960 }
961
962 if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
963 *pbMsrBitmap |= RT_BIT(uMsrpmBit + 1);
964 else
965 {
966 if (!fInNestedGuestMode)
967 *pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
968#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
969 else
970 {
971 /* Only clear the bit if the nested-guest is also not intercepting the MSR write.*/
972 if (!(pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm] & RT_BIT(uMsrpmBit + 1)))
973 *pbMsrBitmap &= ~RT_BIT(uMsrpmBit + 1);
974 else
975 Assert(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
976 }
977#endif
978 }
979}
980
981
982/**
983 * Sets up AMD-V for the specified VM.
984 * This function is only called once per-VM during initalization.
985 *
986 * @returns VBox status code.
987 * @param pVM The cross context VM structure.
988 */
989VMMR0DECL(int) SVMR0SetupVM(PVMCC pVM)
990{
991 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
992 AssertReturn(pVM, VERR_INVALID_PARAMETER);
993
994 /*
995 * Validate and copy over some parameters.
996 */
997 AssertReturn(pVM->hm.s.svm.fSupported, VERR_INCOMPATIBLE_CONFIG);
998 bool const fNestedPaging = pVM->hm.s.fNestedPagingCfg;
999 AssertReturn(!fNestedPaging || (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_NESTED_PAGING), VERR_INCOMPATIBLE_CONFIG);
1000 pVM->hmr0.s.fNestedPaging = fNestedPaging;
1001 pVM->hmr0.s.fAllow64BitGuests = pVM->hm.s.fAllow64BitGuestsCfg;
1002
1003 /*
1004 * Determin some configuration parameters.
1005 */
1006 bool const fPauseFilter = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
1007 bool const fPauseFilterThreshold = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
1008 bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter;
1009
1010 bool const fLbrVirt = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
1011 bool const fUseLbrVirt = fLbrVirt && pVM->hm.s.svm.fLbrVirt; /** @todo IEM implementation etc. */
1012
1013#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1014 bool const fVirtVmsaveVmload = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
1015 bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && fNestedPaging;
1016
1017 bool const fVGif = RT_BOOL(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF);
1018 bool const fUseVGif = fVGif && pVM->hm.s.svm.fVGif;
1019#endif
1020
1021 PVMCPUCC pVCpu0 = VMCC_GET_CPU_0(pVM);
1022 PSVMVMCB pVmcb0 = pVCpu0->hmr0.s.svm.pVmcb;
1023 AssertMsgReturn(RT_VALID_PTR(pVmcb0), ("Invalid pVmcb (%p) for vcpu[0]\n", pVmcb0), VERR_SVM_INVALID_PVMCB);
1024 PSVMVMCBCTRL pVmcbCtrl0 = &pVmcb0->ctrl;
1025
1026 /* Always trap #AC for reasons of security. */
1027 pVmcbCtrl0->u32InterceptXcpt |= RT_BIT_32(X86_XCPT_AC);
1028
1029 /* Always trap #DB for reasons of security. */
1030 pVmcbCtrl0->u32InterceptXcpt |= RT_BIT_32(X86_XCPT_DB);
1031
1032 /* Trap exceptions unconditionally (debug purposes). */
1033#ifdef HMSVM_ALWAYS_TRAP_PF
1034 pVmcbCtrl0->u32InterceptXcpt |= RT_BIT_32(X86_XCPT_PF);
1035#endif
1036#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1037 /* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
1038 pVmcbCtrl0->u32InterceptXcpt |= RT_BIT_32(X86_XCPT_BP)
1039 | RT_BIT_32(X86_XCPT_DE)
1040 | RT_BIT_32(X86_XCPT_NM)
1041 | RT_BIT_32(X86_XCPT_UD)
1042 | RT_BIT_32(X86_XCPT_NP)
1043 | RT_BIT_32(X86_XCPT_SS)
1044 | RT_BIT_32(X86_XCPT_GP)
1045 | RT_BIT_32(X86_XCPT_PF)
1046 | RT_BIT_32(X86_XCPT_MF)
1047 ;
1048#endif
1049
1050 /* Apply the exceptions intercepts needed by the GIM provider. */
1051 if (pVCpu0->hm.s.fGIMTrapXcptUD || pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1052 pVmcbCtrl0->u32InterceptXcpt |= RT_BIT(X86_XCPT_UD);
1053
1054 /* The mesa 3d driver hack needs #GP. */
1055 if (pVCpu0->hm.s.fTrapXcptGpForLovelyMesaDrv)
1056 pVmcbCtrl0->u32InterceptXcpt |= RT_BIT(X86_XCPT_GP);
1057
1058 /* Set up unconditional intercepts and conditions. */
1059 pVmcbCtrl0->u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS
1060 | SVM_CTRL_INTERCEPT_VMMCALL
1061 | SVM_CTRL_INTERCEPT_VMSAVE
1062 | SVM_CTRL_INTERCEPT_VMLOAD
1063 | SVM_CTRL_INTERCEPT_CLGI
1064 | SVM_CTRL_INTERCEPT_STGI;
1065
1066#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
1067 pVmcbCtrl0->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_TASK_SWITCH;
1068#endif
1069
1070#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1071 if (pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm)
1072 {
1073 /* Virtualized VMSAVE/VMLOAD. */
1074 if (fUseVirtVmsaveVmload)
1075 {
1076 pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload = 1;
1077 pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_VMSAVE
1078 | SVM_CTRL_INTERCEPT_VMLOAD);
1079 }
1080 else
1081 Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1082
1083 /* Virtual GIF. */
1084 if (fUseVGif)
1085 {
1086 pVmcbCtrl0->IntCtrl.n.u1VGifEnable = 1;
1087 pVmcbCtrl0->u64InterceptCtrl &= ~( SVM_CTRL_INTERCEPT_CLGI
1088 | SVM_CTRL_INTERCEPT_STGI);
1089 }
1090 else
1091 Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1092 }
1093 else
1094#endif
1095 {
1096 Assert(!pVCpu0->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvm);
1097 Assert(!pVmcbCtrl0->LbrVirt.n.u1VirtVmsaveVmload);
1098 Assert(!pVmcbCtrl0->IntCtrl.n.u1VGifEnable);
1099 }
1100
1101 /* CR4 writes must always be intercepted for tracking PGM mode changes. */
1102 pVmcbCtrl0->u16InterceptWrCRx = RT_BIT(4);
1103
1104 /* Intercept all DRx reads and writes by default. Changed later on. */
1105 pVmcbCtrl0->u16InterceptRdDRx = 0xffff;
1106 pVmcbCtrl0->u16InterceptWrDRx = 0xffff;
1107
1108 /* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
1109 pVmcbCtrl0->IntCtrl.n.u1VIntrMasking = 1;
1110
1111 /* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
1112 and we currently deliver both PIC and APIC interrupts alike, see hmR0SvmEvaluatePendingEvent() */
1113 pVmcbCtrl0->IntCtrl.n.u1IgnoreTPR = 1;
1114
1115 /* Set the IO permission bitmap physical addresses. */
1116 pVmcbCtrl0->u64IOPMPhysAddr = g_HCPhysIOBitmap;
1117
1118 /* LBR virtualization. */
1119 pVmcbCtrl0->LbrVirt.n.u1LbrVirt = fUseLbrVirt;
1120
1121 /* The host ASID MBZ, for the guest start with 1. */
1122 pVmcbCtrl0->TLBCtrl.n.u32ASID = 1;
1123
1124 /* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
1125 pVmcbCtrl0->NestedPagingCtrl.n.u1NestedPaging = fNestedPaging;
1126
1127 /* Without Nested Paging, we need additionally intercepts. */
1128 if (!fNestedPaging)
1129 {
1130 /* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
1131 pVmcbCtrl0->u16InterceptRdCRx |= RT_BIT(3);
1132 pVmcbCtrl0->u16InterceptWrCRx |= RT_BIT(3);
1133
1134 /* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
1135 pVmcbCtrl0->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_INVLPG
1136 | SVM_CTRL_INTERCEPT_TASK_SWITCH;
1137
1138 /* Page faults must be intercepted to implement shadow paging. */
1139 pVmcbCtrl0->u32InterceptXcpt |= RT_BIT(X86_XCPT_PF);
1140 }
1141
1142 /* Workaround for missing OS/2 TLB flush, see ticketref:20625. */
1143 if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
1144 pVmcbCtrl0->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_TR_WRITES;
1145
1146 /* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
1147 if (fUsePauseFilter)
1148 {
1149 Assert(pVM->hm.s.svm.cPauseFilter > 0);
1150 pVmcbCtrl0->u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
1151 if (fPauseFilterThreshold)
1152 pVmcbCtrl0->u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
1153 pVmcbCtrl0->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_PAUSE;
1154 }
1155
1156 /*
1157 * Setup the MSR permission bitmap.
1158 * The following MSRs are saved/restored automatically during the world-switch.
1159 * Don't intercept guest read/write accesses to these MSRs.
1160 */
1161 uint8_t *pbMsrBitmap0 = (uint8_t *)pVCpu0->hmr0.s.svm.pvMsrBitmap;
1162 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1163 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1164 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1165 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1166 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1167 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1168 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1169 if (!pVCpu0->hm.s.svm.fEmulateLongModeSysEnterExit)
1170 {
1171 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1172 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1173 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1174 }
1175 else
1176 {
1177 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_CS, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1178 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1179 hmR0SvmSetMsrPermission(pVCpu0, pbMsrBitmap0, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
1180 }
1181 pVmcbCtrl0->u64MSRPMPhysAddr = pVCpu0->hmr0.s.svm.HCPhysMsrBitmap;
1182
1183 /* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1184 Assert(pVmcbCtrl0->u32VmcbCleanBits == 0);
1185
1186 for (VMCPUID idCpu = 1; idCpu < pVM->cCpus; idCpu++)
1187 {
1188 PVMCPUCC pVCpuCur = VMCC_GET_CPU(pVM, idCpu);
1189 PSVMVMCB pVmcbCur = pVCpuCur->hmr0.s.svm.pVmcb;
1190 AssertMsgReturn(RT_VALID_PTR(pVmcbCur), ("Invalid pVmcb (%p) for vcpu[%u]\n", pVmcbCur, idCpu), VERR_SVM_INVALID_PVMCB);
1191 PSVMVMCBCTRL pVmcbCtrlCur = &pVmcbCur->ctrl;
1192
1193 /* Copy the VMCB control area. */
1194 memcpy(pVmcbCtrlCur, pVmcbCtrl0, sizeof(*pVmcbCtrlCur));
1195
1196 /* Copy the MSR bitmap and setup the VCPU-specific host physical address. */
1197 uint8_t *pbMsrBitmapCur = (uint8_t *)pVCpuCur->hmr0.s.svm.pvMsrBitmap;
1198 memcpy(pbMsrBitmapCur, pbMsrBitmap0, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
1199 pVmcbCtrlCur->u64MSRPMPhysAddr = pVCpuCur->hmr0.s.svm.HCPhysMsrBitmap;
1200
1201 /* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
1202 Assert(pVmcbCtrlCur->u32VmcbCleanBits == 0);
1203
1204 /* Verify our assumption that GIM providers trap #UD uniformly across VCPUs initially. */
1205 Assert(pVCpuCur->hm.s.fGIMTrapXcptUD == pVCpu0->hm.s.fGIMTrapXcptUD);
1206 }
1207
1208#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1209 LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool fUseVGif=%RTbool fUseVirtVmsaveVmload=%RTbool\n", fUsePauseFilter,
1210 fUseLbrVirt, fUseVGif, fUseVirtVmsaveVmload));
1211#else
1212 LogRel(("HM: fUsePauseFilter=%RTbool fUseLbrVirt=%RTbool\n", fUsePauseFilter, fUseLbrVirt));
1213#endif
1214 return VINF_SUCCESS;
1215}
1216
1217
1218/**
1219 * Gets a pointer to the currently active guest (or nested-guest) VMCB.
1220 *
1221 * @returns Pointer to the current context VMCB.
1222 * @param pVCpu The cross context virtual CPU structure.
1223 */
1224DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPUCC pVCpu)
1225{
1226#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1227 if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1228 return &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
1229#endif
1230 return pVCpu->hmr0.s.svm.pVmcb;
1231}
1232
1233
1234/**
1235 * Gets a pointer to the nested-guest VMCB cache.
1236 *
1237 * @returns Pointer to the nested-guest VMCB cache.
1238 * @param pVCpu The cross context virtual CPU structure.
1239 */
1240DECLINLINE(PSVMNESTEDVMCBCACHE) hmR0SvmGetNestedVmcbCache(PVMCPUCC pVCpu)
1241{
1242#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1243 Assert(pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
1244 return &pVCpu->hm.s.svm.NstGstVmcbCache;
1245#else
1246 RT_NOREF(pVCpu);
1247 return NULL;
1248#endif
1249}
1250
1251
1252/**
1253 * Invalidates a guest page by guest virtual address.
1254 *
1255 * @returns VBox status code.
1256 * @param pVCpu The cross context virtual CPU structure.
1257 * @param GCVirt Guest virtual address of the page to invalidate.
1258 */
1259VMMR0DECL(int) SVMR0InvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCVirt)
1260{
1261 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
1262
1263 bool const fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH) || pVCpu->CTX_SUFF(pVM)->hmr0.s.svm.fAlwaysFlushTLB;
1264
1265 /* Skip it if a TLB flush is already pending. */
1266 if (!fFlushPending)
1267 {
1268 Log4Func(("%#RGv\n", GCVirt));
1269
1270 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
1271 AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1272
1273 SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1274 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1275 }
1276 return VINF_SUCCESS;
1277}
1278
1279
1280/**
1281 * Flushes the appropriate tagged-TLB entries.
1282 *
1283 * @param pHostCpu The HM physical-CPU structure.
1284 * @param pVCpu The cross context virtual CPU structure.
1285 * @param pVmcb Pointer to the VM control block.
1286 */
1287static void hmR0SvmFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1288{
1289 /*
1290 * Force a TLB flush for the first world switch if the current CPU differs from the one
1291 * we ran on last. This can happen both for start & resume due to long jumps back to
1292 * ring-3.
1293 *
1294 * We also force a TLB flush every time when executing a nested-guest VCPU as there is no
1295 * correlation between it and the physical CPU.
1296 *
1297 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while
1298 * flushing the TLB, so we cannot reuse the ASIDs without flushing.
1299 */
1300 bool fNewAsid = false;
1301 Assert(pHostCpu->idCpu != NIL_RTCPUID);
1302 if ( pVCpu->hmr0.s.idLastCpu != pHostCpu->idCpu
1303 || pVCpu->hmr0.s.cTlbFlushes != pHostCpu->cTlbFlushes
1304#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1305 || CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx)
1306#endif
1307 )
1308 {
1309 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1310 pVCpu->hmr0.s.fForceTLBFlush = true;
1311 fNewAsid = true;
1312 }
1313
1314 /* Set TLB flush state as checked until we return from the world switch. */
1315 ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1316
1317 /* Check for explicit TLB flushes. */
1318 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1319 {
1320 pVCpu->hmr0.s.fForceTLBFlush = true;
1321 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1322 }
1323
1324 /*
1325 * If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1326 * This Host CPU requirement takes precedence.
1327 */
1328 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1329 if (pVM->hmr0.s.svm.fAlwaysFlushTLB)
1330 {
1331 pHostCpu->uCurrentAsid = 1;
1332 pVCpu->hmr0.s.uCurrentAsid = 1;
1333 pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1334 pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1335 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1336
1337 /* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1338 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1339 }
1340 else
1341 {
1342 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1343 if (pVCpu->hmr0.s.fForceTLBFlush)
1344 {
1345 /* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1346 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1347
1348 if (fNewAsid)
1349 {
1350 ++pHostCpu->uCurrentAsid;
1351
1352 bool fHitASIDLimit = false;
1353 if (pHostCpu->uCurrentAsid >= g_uHmMaxAsid)
1354 {
1355 pHostCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1356 pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1357 fHitASIDLimit = true;
1358 }
1359
1360 if ( fHitASIDLimit
1361 || pHostCpu->fFlushAsidBeforeUse)
1362 {
1363 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1364 pHostCpu->fFlushAsidBeforeUse = false;
1365 }
1366
1367 pVCpu->hmr0.s.uCurrentAsid = pHostCpu->uCurrentAsid;
1368 pVCpu->hmr0.s.idLastCpu = pHostCpu->idCpu;
1369 pVCpu->hmr0.s.cTlbFlushes = pHostCpu->cTlbFlushes;
1370 }
1371 else
1372 {
1373 if (g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1374 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1375 else
1376 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1377 }
1378
1379 pVCpu->hmr0.s.fForceTLBFlush = false;
1380 }
1381 }
1382
1383 /* Update VMCB with the ASID. */
1384 if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hmr0.s.uCurrentAsid)
1385 {
1386 pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hmr0.s.uCurrentAsid;
1387 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1388 }
1389
1390 AssertMsg(pVCpu->hmr0.s.idLastCpu == pHostCpu->idCpu,
1391 ("vcpu idLastCpu=%u hostcpu idCpu=%u\n", pVCpu->hmr0.s.idLastCpu, pHostCpu->idCpu));
1392 AssertMsg(pVCpu->hmr0.s.cTlbFlushes == pHostCpu->cTlbFlushes,
1393 ("Flush count mismatch for cpu %u (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hmr0.s.cTlbFlushes, pHostCpu->cTlbFlushes));
1394 AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < g_uHmMaxAsid,
1395 ("cpu%d uCurrentAsid = %x\n", pHostCpu->idCpu, pHostCpu->uCurrentAsid));
1396 AssertMsg(pVCpu->hmr0.s.uCurrentAsid >= 1 && pVCpu->hmr0.s.uCurrentAsid < g_uHmMaxAsid,
1397 ("cpu%d VM uCurrentAsid = %x\n", pHostCpu->idCpu, pVCpu->hmr0.s.uCurrentAsid));
1398
1399#ifdef VBOX_WITH_STATISTICS
1400 if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1401 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1402 else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1403 || pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1404 {
1405 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1406 }
1407 else
1408 {
1409 Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1410 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1411 }
1412#endif
1413}
1414
1415
1416/**
1417 * Sets an exception intercept in the specified VMCB.
1418 *
1419 * @param pVmcb Pointer to the VM control block.
1420 * @param uXcpt The exception (X86_XCPT_*).
1421 */
1422DECLINLINE(void) hmR0SvmSetXcptIntercept(PSVMVMCB pVmcb, uint8_t uXcpt)
1423{
1424 if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt)))
1425 {
1426 pVmcb->ctrl.u32InterceptXcpt |= RT_BIT(uXcpt);
1427 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1428 }
1429}
1430
1431
1432/**
1433 * Clears an exception intercept in the specified VMCB.
1434 *
1435 * @param pVCpu The cross context virtual CPU structure.
1436 * @param pVmcb Pointer to the VM control block.
1437 * @param uXcpt The exception (X86_XCPT_*).
1438 *
1439 * @remarks This takes into account if we're executing a nested-guest and only
1440 * removes the exception intercept if both the guest -and- nested-guest
1441 * are not intercepting it.
1442 */
1443DECLINLINE(void) hmR0SvmClearXcptIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint8_t uXcpt)
1444{
1445 Assert(uXcpt != X86_XCPT_DB);
1446 Assert(uXcpt != X86_XCPT_AC);
1447 Assert(uXcpt != X86_XCPT_GP);
1448#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1449 if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(uXcpt))
1450 {
1451 bool fRemove = true;
1452# ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1453 /* Only remove the intercept if the nested-guest is also not intercepting it! */
1454 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1455 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1456 {
1457 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1458 fRemove = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(uXcpt));
1459 }
1460# else
1461 RT_NOREF(pVCpu);
1462# endif
1463 if (fRemove)
1464 {
1465 pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(uXcpt);
1466 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1467 }
1468 }
1469#else
1470 RT_NOREF3(pVCpu, pVmcb, uXcpt);
1471#endif
1472}
1473
1474
1475/**
1476 * Sets a control intercept in the specified VMCB.
1477 *
1478 * @param pVmcb Pointer to the VM control block.
1479 * @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1480 */
1481DECLINLINE(void) hmR0SvmSetCtrlIntercept(PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1482{
1483 if (!(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept))
1484 {
1485 pVmcb->ctrl.u64InterceptCtrl |= fCtrlIntercept;
1486 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1487 }
1488}
1489
1490
1491/**
1492 * Clears a control intercept in the specified VMCB.
1493 *
1494 * @returns @c true if the intercept is still set, @c false otherwise.
1495 * @param pVCpu The cross context virtual CPU structure.
1496 * @param pVmcb Pointer to the VM control block.
1497 * @param fCtrlIntercept The control intercept (SVM_CTRL_INTERCEPT_*).
1498 *
1499 * @remarks This takes into account if we're executing a nested-guest and only
1500 * removes the control intercept if both the guest -and- nested-guest
1501 * are not intercepting it.
1502 */
1503static bool hmR0SvmClearCtrlIntercept(PVMCPUCC pVCpu, PSVMVMCB pVmcb, uint64_t fCtrlIntercept)
1504{
1505 if (pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept)
1506 {
1507 bool fRemove = true;
1508#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
1509 /* Only remove the control intercept if the nested-guest is also not intercepting it! */
1510 if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
1511 {
1512 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1513 fRemove = !(pVmcbNstGstCache->u64InterceptCtrl & fCtrlIntercept);
1514 }
1515#else
1516 RT_NOREF(pVCpu);
1517#endif
1518 if (fRemove)
1519 {
1520 pVmcb->ctrl.u64InterceptCtrl &= ~fCtrlIntercept;
1521 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1522 }
1523 }
1524
1525 return RT_BOOL(pVmcb->ctrl.u64InterceptCtrl & fCtrlIntercept);
1526}
1527
1528
1529/**
1530 * Exports the guest (or nested-guest) CR0 into the VMCB.
1531 *
1532 * @param pVCpu The cross context virtual CPU structure.
1533 * @param pVmcb Pointer to the VM control block.
1534 *
1535 * @remarks This assumes we always pre-load the guest FPU.
1536 * @remarks No-long-jump zone!!!
1537 */
1538static void hmR0SvmExportGuestCR0(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1539{
1540 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1541
1542 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1543 uint64_t const uGuestCr0 = pCtx->cr0;
1544 uint64_t uShadowCr0 = uGuestCr0;
1545
1546 /* Always enable caching. */
1547 uShadowCr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1548
1549 /* When Nested Paging is not available use shadow page tables and intercept #PFs (latter done in SVMR0SetupVM()). */
1550 if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1551 {
1552 uShadowCr0 |= X86_CR0_PG /* Use shadow page tables. */
1553 | X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1554 }
1555
1556 /*
1557 * Use the #MF style of legacy-FPU error reporting for now. Although AMD-V has MSRs that
1558 * lets us isolate the host from it, IEM/REM still needs work to emulate it properly,
1559 * see @bugref{7243#c103}.
1560 */
1561 if (!(uGuestCr0 & X86_CR0_NE))
1562 {
1563 uShadowCr0 |= X86_CR0_NE;
1564 hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_MF);
1565 }
1566 else
1567 hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_MF);
1568
1569 /*
1570 * If the shadow and guest CR0 are identical we can avoid intercepting CR0 reads.
1571 *
1572 * CR0 writes still needs interception as PGM requires tracking paging mode changes,
1573 * see @bugref{6944}.
1574 *
1575 * We also don't ever want to honor weird things like cache disable from the guest.
1576 * However, we can avoid intercepting changes to the TS & MP bits by clearing the CR0
1577 * write intercept below and keeping SVM_CTRL_INTERCEPT_CR0_SEL_WRITE instead.
1578 */
1579 if (uShadowCr0 == uGuestCr0)
1580 {
1581 if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1582 {
1583 pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(0);
1584 pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(0);
1585 Assert(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_CR0_SEL_WRITE);
1586 }
1587 else
1588 {
1589 /* If the nested-hypervisor intercepts CR0 reads/writes, we need to continue intercepting them. */
1590 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1591 pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(0))
1592 | (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(0));
1593 pVmcb->ctrl.u16InterceptWrCRx = (pVmcb->ctrl.u16InterceptWrCRx & ~RT_BIT(0))
1594 | (pVmcbNstGstCache->u16InterceptWrCRx & RT_BIT(0));
1595 }
1596 }
1597 else
1598 {
1599 pVmcb->ctrl.u16InterceptRdCRx |= RT_BIT(0);
1600 pVmcb->ctrl.u16InterceptWrCRx |= RT_BIT(0);
1601 }
1602 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1603
1604 Assert(!RT_HI_U32(uShadowCr0));
1605 if (pVmcb->guest.u64CR0 != uShadowCr0)
1606 {
1607 pVmcb->guest.u64CR0 = uShadowCr0;
1608 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1609 }
1610}
1611
1612
1613/**
1614 * Exports the guest (or nested-guest) CR3 into the VMCB.
1615 *
1616 * @param pVCpu The cross context virtual CPU structure.
1617 * @param pVmcb Pointer to the VM control block.
1618 *
1619 * @remarks No-long-jump zone!!!
1620 */
1621static void hmR0SvmExportGuestCR3(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1622{
1623 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1624
1625 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1626 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1627 if (pVM->hmr0.s.fNestedPaging)
1628 {
1629 pVmcb->ctrl.u64NestedPagingCR3 = PGMGetHyperCR3(pVCpu);
1630 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1631 pVmcb->guest.u64CR3 = pCtx->cr3;
1632 Assert(pVmcb->ctrl.u64NestedPagingCR3);
1633 }
1634 else
1635 pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1636
1637 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1638}
1639
1640
1641/**
1642 * Exports the guest (or nested-guest) CR4 into the VMCB.
1643 *
1644 * @param pVCpu The cross context virtual CPU structure.
1645 * @param pVmcb Pointer to the VM control block.
1646 *
1647 * @remarks No-long-jump zone!!!
1648 */
1649static int hmR0SvmExportGuestCR4(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1650{
1651 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1652
1653 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1654 uint64_t uShadowCr4 = pCtx->cr4;
1655 if (!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging)
1656 {
1657 switch (pVCpu->hm.s.enmShadowMode)
1658 {
1659 case PGMMODE_REAL:
1660 case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1661 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1662
1663 case PGMMODE_32_BIT: /* 32-bit paging. */
1664 uShadowCr4 &= ~X86_CR4_PAE;
1665 break;
1666
1667 case PGMMODE_PAE: /* PAE paging. */
1668 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1669 /** Must use PAE paging as we could use physical memory > 4 GB */
1670 uShadowCr4 |= X86_CR4_PAE;
1671 break;
1672
1673 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1674 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1675#ifdef VBOX_WITH_64_BITS_GUESTS
1676 break;
1677#else
1678 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1679#endif
1680
1681 default: /* shut up gcc */
1682 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1683 }
1684 }
1685
1686 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1687 bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1688 if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
1689 {
1690 pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
1691 hmR0SvmUpdateVmRunFunction(pVCpu);
1692 }
1693
1694 /* Avoid intercepting CR4 reads if the guest and shadow CR4 values are identical. */
1695 if (uShadowCr4 == pCtx->cr4)
1696 {
1697 if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1698 pVmcb->ctrl.u16InterceptRdCRx &= ~RT_BIT(4);
1699 else
1700 {
1701 /* If the nested-hypervisor intercepts CR4 reads, we need to continue intercepting them. */
1702 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = hmR0SvmGetNestedVmcbCache(pVCpu);
1703 pVmcb->ctrl.u16InterceptRdCRx = (pVmcb->ctrl.u16InterceptRdCRx & ~RT_BIT(4))
1704 | (pVmcbNstGstCache->u16InterceptRdCRx & RT_BIT(4));
1705 }
1706 }
1707 else
1708 pVmcb->ctrl.u16InterceptRdCRx |= RT_BIT(4);
1709
1710 /* CR4 writes are always intercepted (both guest, nested-guest) for tracking PGM mode changes. */
1711 Assert(pVmcb->ctrl.u16InterceptWrCRx & RT_BIT(4));
1712
1713 /* Update VMCB with the shadow CR4 the appropriate VMCB clean bits. */
1714 Assert(!RT_HI_U32(uShadowCr4));
1715 pVmcb->guest.u64CR4 = uShadowCr4;
1716 pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_CRX_EFER | HMSVM_VMCB_CLEAN_INTERCEPTS);
1717
1718 return VINF_SUCCESS;
1719}
1720
1721
1722/**
1723 * Exports the guest (or nested-guest) control registers into the VMCB.
1724 *
1725 * @returns VBox status code.
1726 * @param pVCpu The cross context virtual CPU structure.
1727 * @param pVmcb Pointer to the VM control block.
1728 *
1729 * @remarks No-long-jump zone!!!
1730 */
1731static int hmR0SvmExportGuestControlRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1732{
1733 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1734
1735 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR_MASK)
1736 {
1737 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR0)
1738 hmR0SvmExportGuestCR0(pVCpu, pVmcb);
1739
1740 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR2)
1741 {
1742 pVmcb->guest.u64CR2 = pVCpu->cpum.GstCtx.cr2;
1743 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1744 }
1745
1746 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR3)
1747 hmR0SvmExportGuestCR3(pVCpu, pVmcb);
1748
1749 /* CR4 re-loading is ASSUMED to be done everytime we get in from ring-3! (XCR0) */
1750 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CR4)
1751 {
1752 int rc = hmR0SvmExportGuestCR4(pVCpu, pVmcb);
1753 if (RT_FAILURE(rc))
1754 return rc;
1755 }
1756
1757 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_CR_MASK;
1758 }
1759 return VINF_SUCCESS;
1760}
1761
1762
1763/**
1764 * Exports the guest (or nested-guest) segment registers into the VMCB.
1765 *
1766 * @returns VBox status code.
1767 * @param pVCpu The cross context virtual CPU structure.
1768 * @param pVmcb Pointer to the VM control block.
1769 *
1770 * @remarks No-long-jump zone!!!
1771 */
1772static void hmR0SvmExportGuestSegmentRegs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1773{
1774 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1775 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1776
1777 /* Guest segment registers. */
1778 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SREG_MASK)
1779 {
1780 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_CS)
1781 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1782
1783 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SS)
1784 {
1785 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1786 pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1787 }
1788
1789 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DS)
1790 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1791
1792 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_ES)
1793 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1794
1795 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_FS)
1796 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1797
1798 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GS)
1799 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1800
1801 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1802 }
1803
1804 /* Guest TR. */
1805 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_TR)
1806 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1807
1808 /* Guest LDTR. */
1809 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_LDTR)
1810 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1811
1812 /* Guest GDTR. */
1813 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_GDTR)
1814 {
1815 pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1816 pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1817 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1818 }
1819
1820 /* Guest IDTR. */
1821 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_IDTR)
1822 {
1823 pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1824 pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1825 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1826 }
1827
1828 pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SREG_MASK
1829 | HM_CHANGED_GUEST_TABLE_MASK);
1830}
1831
1832
1833/**
1834 * Exports the guest (or nested-guest) MSRs into the VMCB.
1835 *
1836 * @param pVCpu The cross context virtual CPU structure.
1837 * @param pVmcb Pointer to the VM control block.
1838 *
1839 * @remarks No-long-jump zone!!!
1840 */
1841static void hmR0SvmExportGuestMsrs(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1842{
1843 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1844 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1845
1846 /* Guest Sysenter MSRs. */
1847 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
1848 {
1849 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
1850 pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1851
1852 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
1853 pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1854
1855 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
1856 pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1857 }
1858
1859 /*
1860 * Guest EFER MSR.
1861 * AMD-V requires guest EFER.SVME to be set. Weird.
1862 * See AMD spec. 15.5.1 "Basic Operation" | "Canonicalization and Consistency Checks".
1863 */
1864 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_EFER_MSR)
1865 {
1866 pVmcb->guest.u64EFER = pCtx->msrEFER | MSR_K6_EFER_SVME;
1867 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1868 }
1869
1870 /* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit, otherwise SVM expects amd64 shadow paging. */
1871 if ( !CPUMIsGuestInLongModeEx(pCtx)
1872 && (pCtx->msrEFER & MSR_K6_EFER_LME))
1873 {
1874 pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1875 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1876 }
1877
1878 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_SYSCALL_MSRS)
1879 {
1880 pVmcb->guest.u64STAR = pCtx->msrSTAR;
1881 pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1882 pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1883 pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1884 }
1885
1886 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_KERNEL_GS_BASE)
1887 pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1888
1889 pVCpu->hm.s.fCtxChanged &= ~( HM_CHANGED_GUEST_SYSENTER_MSR_MASK
1890 | HM_CHANGED_GUEST_EFER_MSR
1891 | HM_CHANGED_GUEST_SYSCALL_MSRS
1892 | HM_CHANGED_GUEST_KERNEL_GS_BASE);
1893
1894 /*
1895 * Setup the PAT MSR (applicable for Nested Paging only).
1896 *
1897 * The default value should be MSR_IA32_CR_PAT_INIT_VAL, but we treat all guest memory
1898 * as WB, so choose type 6 for all PAT slots, see @bugref{9634}.
1899 *
1900 * While guests can modify and see the modified values through the shadow values,
1901 * we shall not honor any guest modifications of this MSR to ensure caching is always
1902 * enabled similar to how we clear CR0.CD and NW bits.
1903 *
1904 * For nested-guests this needs to always be set as well, see @bugref{7243#c109}.
1905 */
1906 pVmcb->guest.u64PAT = UINT64_C(0x0006060606060606);
1907
1908 /* Enable the last branch record bit if LBR virtualization is enabled. */
1909 if (pVmcb->ctrl.LbrVirt.n.u1LbrVirt)
1910 pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
1911}
1912
1913
1914/**
1915 * Exports the guest (or nested-guest) debug state into the VMCB and programs
1916 * the necessary intercepts accordingly.
1917 *
1918 * @param pVCpu The cross context virtual CPU structure.
1919 * @param pVmcb Pointer to the VM control block.
1920 *
1921 * @remarks No-long-jump zone!!!
1922 * @remarks Requires EFLAGS to be up-to-date in the VMCB!
1923 */
1924static void hmR0SvmExportSharedDebugState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
1925{
1926 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1927
1928 /** @todo Figure out stepping with nested-guest. */
1929 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1930 {
1931 /*
1932 * We don't want to always intercept DRx read/writes for nested-guests as it causes
1933 * problems when the nested hypervisor isn't intercepting them, see @bugref{10080}.
1934 * Instead, they are strictly only requested when the nested hypervisor intercepts
1935 * them -- handled while merging VMCB controls.
1936 *
1937 * If neither the outer nor the nested-hypervisor is intercepting DRx read/writes,
1938 * then the nested-guest debug state should be actively loaded on the host so that
1939 * nested-guest reads/writes its own debug registers without causing VM-exits.
1940 */
1941 if ( ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1942 || pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1943 && !CPUMIsGuestDebugStateActive(pVCpu))
1944 {
1945 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
1946 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1947 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1948 Assert(CPUMIsGuestDebugStateActive(pVCpu));
1949 }
1950
1951 pVmcb->guest.u64DR6 = pCtx->dr[6];
1952 pVmcb->guest.u64DR7 = pCtx->dr[7];
1953 return;
1954 }
1955
1956 /*
1957 * Anyone single stepping on the host side? If so, we'll have to use the
1958 * trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1959 * the VMM level like the VT-x implementations does.
1960 */
1961 bool fInterceptMovDRx = false;
1962 bool const fStepping = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
1963 if (fStepping)
1964 {
1965 pVCpu->hmr0.s.fClearTrapFlag = true;
1966 pVmcb->guest.u64RFlags |= X86_EFL_TF;
1967 fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1968 }
1969
1970 if ( fStepping
1971 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1972 {
1973 /*
1974 * Use the combined guest and host DRx values found in the hypervisor
1975 * register set because the debugger has breakpoints active or someone
1976 * is single stepping on the host side.
1977 *
1978 * Note! DBGF expects a clean DR6 state before executing guest code.
1979 */
1980 if (!CPUMIsHyperDebugStateActive(pVCpu))
1981 {
1982 CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1983 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1984 Assert(CPUMIsHyperDebugStateActive(pVCpu));
1985 }
1986
1987 /* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1988 if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1989 || pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1990 {
1991 pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1992 pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1993 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1994 }
1995
1996 /** @todo If we cared, we could optimize to allow the guest to read registers
1997 * with the same values. */
1998 fInterceptMovDRx = true;
1999 pVCpu->hmr0.s.fUsingHyperDR7 = true;
2000 Log5(("hmR0SvmExportSharedDebugState: Loaded hyper DRx\n"));
2001 }
2002 else
2003 {
2004 /*
2005 * Update DR6, DR7 with the guest values if necessary.
2006 */
2007 if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
2008 || pVmcb->guest.u64DR6 != pCtx->dr[6])
2009 {
2010 pVmcb->guest.u64DR7 = pCtx->dr[7];
2011 pVmcb->guest.u64DR6 = pCtx->dr[6];
2012 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
2013 }
2014 pVCpu->hmr0.s.fUsingHyperDR7 = false;
2015
2016 /*
2017 * If the guest has enabled debug registers, we need to load them prior to
2018 * executing guest code so they'll trigger at the right time.
2019 */
2020 if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
2021 {
2022 if (!CPUMIsGuestDebugStateActive(pVCpu))
2023 {
2024 CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
2025 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2026 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2027 Assert(CPUMIsGuestDebugStateActive(pVCpu));
2028 }
2029 Log5(("hmR0SvmExportSharedDebugState: Loaded guest DRx\n"));
2030 }
2031 /*
2032 * If no debugging enabled, we'll lazy load DR0-3. We don't need to
2033 * intercept #DB as DR6 is updated in the VMCB.
2034 *
2035 * Note! If we cared and dared, we could skip intercepting \#DB here.
2036 * However, \#DB shouldn't be performance critical, so we'll play safe
2037 * and keep the code similar to the VT-x code and always intercept it.
2038 */
2039 else if (!CPUMIsGuestDebugStateActive(pVCpu))
2040 fInterceptMovDRx = true;
2041 }
2042
2043 Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
2044 if (fInterceptMovDRx)
2045 {
2046 if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
2047 || pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
2048 {
2049 pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
2050 pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
2051 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2052 }
2053 }
2054 else
2055 {
2056 if ( pVmcb->ctrl.u16InterceptRdDRx
2057 || pVmcb->ctrl.u16InterceptWrDRx)
2058 {
2059 pVmcb->ctrl.u16InterceptRdDRx = 0;
2060 pVmcb->ctrl.u16InterceptWrDRx = 0;
2061 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2062 }
2063 }
2064 Log4Func(("DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2065}
2066
2067/**
2068 * Exports the hardware virtualization state into the nested-guest
2069 * VMCB.
2070 *
2071 * @param pVCpu The cross context virtual CPU structure.
2072 * @param pVmcb Pointer to the VM control block.
2073 *
2074 * @remarks No-long-jump zone!!!
2075 */
2076static void hmR0SvmExportGuestHwvirtState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2077{
2078 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2079
2080 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_HWVIRT)
2081 {
2082 if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
2083 {
2084 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2085 PCVMCC pVM = pVCpu->CTX_SUFF(pVM);
2086
2087 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx); /* Nested VGIF is not supported yet. */
2088 Assert(g_fHmSvmFeatures & X86_CPUID_SVM_FEATURE_EDX_VGIF); /* Physical hardware supports VGIF. */
2089 Assert(HMIsSvmVGifActive(pVM)); /* Outer VM has enabled VGIF. */
2090 NOREF(pVM);
2091
2092 pVmcb->ctrl.IntCtrl.n.u1VGif = CPUMGetGuestGif(pCtx);
2093 }
2094
2095 /*
2096 * Ensure the nested-guest pause-filter counters don't exceed the outer guest values esp.
2097 * since SVM doesn't have a preemption timer.
2098 *
2099 * We do this here rather than in hmR0SvmSetupVmcbNested() as we may have been executing the
2100 * nested-guest in IEM incl. PAUSE instructions which would update the pause-filter counters
2101 * and may continue execution in SVM R0 without a nested-guest #VMEXIT in between.
2102 */
2103 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2104 PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2105 uint16_t const uGuestPauseFilterCount = pVM->hm.s.svm.cPauseFilter;
2106 uint16_t const uGuestPauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
2107 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, &pVCpu->cpum.GstCtx, SVM_CTRL_INTERCEPT_PAUSE))
2108 {
2109 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2110 pVmcbCtrl->u16PauseFilterCount = RT_MIN(pCtx->hwvirt.svm.cPauseFilter, uGuestPauseFilterCount);
2111 pVmcbCtrl->u16PauseFilterThreshold = RT_MIN(pCtx->hwvirt.svm.cPauseFilterThreshold, uGuestPauseFilterThreshold);
2112 }
2113 else
2114 {
2115 /** @todo r=ramshankar: We can turn these assignments into assertions. */
2116 pVmcbCtrl->u16PauseFilterCount = uGuestPauseFilterCount;
2117 pVmcbCtrl->u16PauseFilterThreshold = uGuestPauseFilterThreshold;
2118 }
2119 pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2120
2121 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_HWVIRT;
2122 }
2123}
2124
2125
2126/**
2127 * Exports the guest APIC TPR state into the VMCB.
2128 *
2129 * @returns VBox status code.
2130 * @param pVCpu The cross context virtual CPU structure.
2131 * @param pVmcb Pointer to the VM control block.
2132 */
2133static int hmR0SvmExportGuestApicTpr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2134{
2135 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2136
2137 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
2138 {
2139 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2140 if ( PDMHasApic(pVM)
2141 && APICIsEnabled(pVCpu))
2142 {
2143 bool fPendingIntr;
2144 uint8_t u8Tpr;
2145 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
2146 AssertRCReturn(rc, rc);
2147
2148 /* Assume that we need to trap all TPR accesses and thus need not check on
2149 every #VMEXIT if we should update the TPR. */
2150 Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
2151 pVCpu->hmr0.s.svm.fSyncVTpr = false;
2152
2153 if (!pVM->hm.s.fTprPatchingActive)
2154 {
2155 /* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2156 pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
2157
2158 /* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we
2159 can deliver the interrupt to the guest. */
2160 if (fPendingIntr)
2161 pVmcb->ctrl.u16InterceptWrCRx |= RT_BIT(8);
2162 else
2163 {
2164 pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
2165 pVCpu->hmr0.s.svm.fSyncVTpr = true;
2166 }
2167
2168 pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS | HMSVM_VMCB_CLEAN_INT_CTRL);
2169 }
2170 else
2171 {
2172 /* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2173 pVmcb->guest.u64LSTAR = u8Tpr;
2174 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
2175
2176 /* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
2177 if (fPendingIntr)
2178 hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
2179 else
2180 {
2181 hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
2182 pVCpu->hmr0.s.svm.fSyncVTpr = true;
2183 }
2184 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
2185 }
2186 }
2187 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
2188 }
2189 return VINF_SUCCESS;
2190}
2191
2192
2193/**
2194 * Sets up the exception interrupts required for guest execution in the VMCB.
2195 *
2196 * @param pVCpu The cross context virtual CPU structure.
2197 * @param pVmcb Pointer to the VM control block.
2198 *
2199 * @remarks No-long-jump zone!!!
2200 */
2201static void hmR0SvmExportGuestXcptIntercepts(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2202{
2203 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2204
2205 /* If we modify intercepts from here, please check & adjust hmR0SvmMergeVmcbCtrlsNested() if required. */
2206 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_SVM_XCPT_INTERCEPTS)
2207 {
2208 /* Trap #UD for GIM provider (e.g. for hypercalls). */
2209 if (pVCpu->hm.s.fGIMTrapXcptUD || pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
2210 hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_UD);
2211 else
2212 hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_UD);
2213
2214 /* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
2215 if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
2216 hmR0SvmSetXcptIntercept(pVmcb, X86_XCPT_BP);
2217 else
2218 hmR0SvmClearXcptIntercept(pVCpu, pVmcb, X86_XCPT_BP);
2219
2220 /* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmExportGuestCR0(). */
2221 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_SVM_XCPT_INTERCEPTS);
2222 }
2223}
2224
2225
2226#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2227/**
2228 * Merges guest and nested-guest intercepts for executing the nested-guest using
2229 * hardware-assisted SVM.
2230 *
2231 * This merges the guest and nested-guest intercepts in a way that if the outer
2232 * guest intercept is set we need to intercept it in the nested-guest as
2233 * well.
2234 *
2235 * @param pVCpu The cross context virtual CPU structure.
2236 * @param pVmcbNstGst Pointer to the nested-guest VM control block.
2237 */
2238static void hmR0SvmMergeVmcbCtrlsNested(PVMCPUCC pVCpu)
2239{
2240 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2241 PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
2242 PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2243 PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2244
2245 /* Merge the guest's CR intercepts into the nested-guest VMCB. */
2246 pVmcbNstGstCtrl->u16InterceptRdCRx |= pVmcb->ctrl.u16InterceptRdCRx;
2247 pVmcbNstGstCtrl->u16InterceptWrCRx |= pVmcb->ctrl.u16InterceptWrCRx;
2248
2249 /* Always intercept CR4 writes for tracking PGM mode changes. */
2250 pVmcbNstGstCtrl->u16InterceptWrCRx |= RT_BIT(4);
2251
2252 /* Without nested paging, intercept CR3 reads and writes as we load shadow page tables. */
2253 if (!pVM->hmr0.s.fNestedPaging)
2254 {
2255 pVmcbNstGstCtrl->u16InterceptRdCRx |= RT_BIT(3);
2256 pVmcbNstGstCtrl->u16InterceptWrCRx |= RT_BIT(3);
2257 }
2258
2259 /* Merge the guest's DR intercepts into the nested-guest VMCB. */
2260 pVmcbNstGstCtrl->u16InterceptRdDRx |= pVmcb->ctrl.u16InterceptRdDRx;
2261 pVmcbNstGstCtrl->u16InterceptWrDRx |= pVmcb->ctrl.u16InterceptWrDRx;
2262
2263 /*
2264 * Merge the guest's exception intercepts into the nested-guest VMCB.
2265 *
2266 * - #UD: Exclude these as the outer guest's GIM hypercalls are not applicable
2267 * while executing the nested-guest.
2268 *
2269 * - #BP: Exclude breakpoints set by the VM debugger for the outer guest. This can
2270 * be tweaked later depending on how we wish to implement breakpoints.
2271 *
2272 * - #GP: Exclude these as it's the inner VMMs problem to get vmsvga 3d drivers
2273 * loaded into their guests, not ours.
2274 *
2275 * Warning!! This ASSUMES we only intercept \#UD for hypercall purposes and \#BP
2276 * for VM debugger breakpoints, see hmR0SvmExportGuestXcptIntercepts().
2277 */
2278#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
2279 pVmcbNstGstCtrl->u32InterceptXcpt |= pVmcb->ctrl.u32InterceptXcpt
2280 & ~( RT_BIT(X86_XCPT_UD)
2281 | RT_BIT(X86_XCPT_BP)
2282 | (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv ? RT_BIT(X86_XCPT_GP) : 0));
2283#else
2284 pVmcbNstGstCtrl->u32InterceptXcpt |= pVmcb->ctrl.u32InterceptXcpt;
2285#endif
2286
2287 /*
2288 * Adjust intercepts while executing the nested-guest that differ from the
2289 * outer guest intercepts.
2290 *
2291 * - VINTR: Exclude the outer guest intercept as we don't need to cause VINTR #VMEXITs
2292 * that belong to the nested-guest to the outer guest.
2293 *
2294 * - VMMCALL: Exclude the outer guest intercept as when it's also not intercepted by
2295 * the nested-guest, the physical CPU raises a \#UD exception as expected.
2296 */
2297 pVmcbNstGstCtrl->u64InterceptCtrl |= (pVmcb->ctrl.u64InterceptCtrl & ~( SVM_CTRL_INTERCEPT_VINTR
2298 | SVM_CTRL_INTERCEPT_VMMCALL))
2299 | HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
2300
2301 Assert( (pVmcbNstGstCtrl->u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
2302 == HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
2303
2304 /* Finally, update the VMCB clean bits. */
2305 pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2306}
2307#endif
2308
2309
2310/**
2311 * Enters the AMD-V session.
2312 *
2313 * @returns VBox status code.
2314 * @param pVCpu The cross context virtual CPU structure.
2315 */
2316VMMR0DECL(int) SVMR0Enter(PVMCPUCC pVCpu)
2317{
2318 AssertPtr(pVCpu);
2319 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.svm.fSupported);
2320 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2321
2322 LogFlowFunc(("pVCpu=%p\n", pVCpu));
2323 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2324 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2325
2326 pVCpu->hmr0.s.fLeaveDone = false;
2327 return VINF_SUCCESS;
2328}
2329
2330
2331/**
2332 * Thread-context callback for AMD-V.
2333 *
2334 * This is used together with RTThreadCtxHookCreate() on platforms which
2335 * supports it, and directly from VMMR0EmtPrepareForBlocking() and
2336 * VMMR0EmtResumeAfterBlocking() on platforms which don't.
2337 *
2338 * @param enmEvent The thread-context event.
2339 * @param pVCpu The cross context virtual CPU structure.
2340 * @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2341 * @thread EMT(pVCpu)
2342 */
2343VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPUCC pVCpu, bool fGlobalInit)
2344{
2345 NOREF(fGlobalInit);
2346
2347 switch (enmEvent)
2348 {
2349 case RTTHREADCTXEVENT_OUT:
2350 {
2351 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2352 VMCPU_ASSERT_EMT(pVCpu);
2353
2354 /* No longjmps (log-flush, locks) in this fragile context. */
2355 VMMRZCallRing3Disable(pVCpu);
2356
2357 if (!pVCpu->hmr0.s.fLeaveDone)
2358 {
2359 hmR0SvmLeave(pVCpu, false /* fImportState */);
2360 pVCpu->hmr0.s.fLeaveDone = true;
2361 }
2362
2363 /* Leave HM context, takes care of local init (term). */
2364 int rc = HMR0LeaveCpu(pVCpu);
2365 AssertRC(rc); NOREF(rc);
2366
2367 /* Restore longjmp state. */
2368 VMMRZCallRing3Enable(pVCpu);
2369 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2370 break;
2371 }
2372
2373 case RTTHREADCTXEVENT_IN:
2374 {
2375 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2376 VMCPU_ASSERT_EMT(pVCpu);
2377
2378 /* No longjmps (log-flush, locks) in this fragile context. */
2379 VMMRZCallRing3Disable(pVCpu);
2380
2381 /*
2382 * Initialize the bare minimum state required for HM. This takes care of
2383 * initializing AMD-V if necessary (onlined CPUs, local init etc.)
2384 */
2385 int rc = hmR0EnterCpu(pVCpu);
2386 AssertRC(rc); NOREF(rc);
2387 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE))
2388 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE));
2389
2390 pVCpu->hmr0.s.fLeaveDone = false;
2391
2392 /* Restore longjmp state. */
2393 VMMRZCallRing3Enable(pVCpu);
2394 break;
2395 }
2396
2397 default:
2398 break;
2399 }
2400}
2401
2402
2403/**
2404 * Saves the host state.
2405 *
2406 * @returns VBox status code.
2407 * @param pVCpu The cross context virtual CPU structure.
2408 *
2409 * @remarks No-long-jump zone!!!
2410 */
2411VMMR0DECL(int) SVMR0ExportHostState(PVMCPUCC pVCpu)
2412{
2413 NOREF(pVCpu);
2414
2415 /* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2416 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_HOST_CONTEXT);
2417 return VINF_SUCCESS;
2418}
2419
2420
2421/**
2422 * Exports the guest or nested-guest state from the virtual-CPU context into the
2423 * VMCB.
2424 *
2425 * Also sets up the appropriate VMRUN function to execute guest or nested-guest
2426 * code based on the virtual-CPU mode.
2427 *
2428 * @returns VBox status code.
2429 * @param pVCpu The cross context virtual CPU structure.
2430 * @param pSvmTransient Pointer to the SVM-transient structure.
2431 *
2432 * @remarks No-long-jump zone!!!
2433 */
2434static int hmR0SvmExportGuestState(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
2435{
2436 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
2437
2438 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2439 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2440 Assert(pVmcb);
2441
2442 pVmcb->guest.u64RIP = pCtx->rip;
2443 pVmcb->guest.u64RSP = pCtx->rsp;
2444 pVmcb->guest.u64RFlags = pCtx->eflags.u32;
2445 pVmcb->guest.u64RAX = pCtx->rax;
2446
2447 bool const fIsNestedGuest = pSvmTransient->fIsNestedGuest;
2448 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2449
2450 int rc = hmR0SvmExportGuestControlRegs(pVCpu, pVmcb);
2451 AssertRCReturnStmt(rc, ASMSetFlags(fEFlags), rc);
2452 hmR0SvmExportGuestSegmentRegs(pVCpu, pVmcb);
2453 hmR0SvmExportGuestMsrs(pVCpu, pVmcb);
2454 hmR0SvmExportGuestHwvirtState(pVCpu, pVmcb);
2455
2456 ASMSetFlags(fEFlags);
2457
2458 if (!fIsNestedGuest)
2459 {
2460 /* hmR0SvmExportGuestApicTpr() must be called -after- hmR0SvmExportGuestMsrs() as we
2461 otherwise we would overwrite the LSTAR MSR that we use for TPR patching. */
2462 hmR0SvmExportGuestApicTpr(pVCpu, pVmcb);
2463 hmR0SvmExportGuestXcptIntercepts(pVCpu, pVmcb);
2464 }
2465
2466 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
2467 uint64_t fUnusedMask = HM_CHANGED_GUEST_RIP
2468 | HM_CHANGED_GUEST_RFLAGS
2469 | HM_CHANGED_GUEST_GPRS_MASK
2470 | HM_CHANGED_GUEST_X87
2471 | HM_CHANGED_GUEST_SSE_AVX
2472 | HM_CHANGED_GUEST_OTHER_XSAVE
2473 | HM_CHANGED_GUEST_XCRx
2474 | HM_CHANGED_GUEST_TSC_AUX
2475 | HM_CHANGED_GUEST_OTHER_MSRS;
2476 if (fIsNestedGuest)
2477 fUnusedMask |= HM_CHANGED_SVM_XCPT_INTERCEPTS
2478 | HM_CHANGED_GUEST_APIC_TPR;
2479
2480 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( fUnusedMask
2481 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_SVM_MASK)));
2482
2483#ifdef VBOX_STRICT
2484 /*
2485 * All of the guest-CPU state and SVM keeper bits should be exported here by now,
2486 * except for the host-context and/or shared host-guest context bits.
2487 */
2488 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
2489 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)),
2490 ("fCtxChanged=%#RX64\n", fCtxChanged));
2491
2492 /*
2493 * If we need to log state that isn't always imported, we'll need to import them here.
2494 * See hmR0SvmPostRunGuest() for which part of the state is imported uncondtionally.
2495 */
2496 hmR0SvmLogState(pVCpu, pVmcb, "hmR0SvmExportGuestState", 0 /* fFlags */, 0 /* uVerbose */);
2497#endif
2498
2499 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
2500 return VINF_SUCCESS;
2501}
2502
2503#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2504
2505/**
2506 * Merges the guest and nested-guest MSR permission bitmap.
2507 *
2508 * If the guest is intercepting an MSR we need to intercept it regardless of
2509 * whether the nested-guest is intercepting it or not.
2510 *
2511 * @param pHostCpu The HM physical-CPU structure.
2512 * @param pVCpu The cross context virtual CPU structure.
2513 *
2514 * @remarks No-long-jmp zone!!!
2515 */
2516DECLINLINE(void) hmR0SvmMergeMsrpmNested(PHMPHYSCPU pHostCpu, PVMCPUCC pVCpu)
2517{
2518 uint64_t const *pu64GstMsrpm = (uint64_t const *)pVCpu->hmr0.s.svm.pvMsrBitmap;
2519 uint64_t const *pu64NstGstMsrpm = (uint64_t const *)&pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[0];
2520 uint64_t *pu64DstMsrpm = (uint64_t *)pHostCpu->n.svm.pvNstGstMsrpm;
2521
2522 /* MSRPM bytes from offset 0x1800 are reserved, so we stop merging there. */
2523 uint32_t const offRsvdQwords = 0x1800 >> 3;
2524 for (uint32_t i = 0; i < offRsvdQwords; i++)
2525 pu64DstMsrpm[i] = pu64NstGstMsrpm[i] | pu64GstMsrpm[i];
2526}
2527
2528
2529/**
2530 * Caches the nested-guest VMCB fields before we modify them for execution using
2531 * hardware-assisted SVM.
2532 *
2533 * @returns true if the VMCB was previously already cached, false otherwise.
2534 * @param pVCpu The cross context virtual CPU structure.
2535 *
2536 * @sa HMNotifySvmNstGstVmexit.
2537 */
2538static bool hmR0SvmCacheVmcbNested(PVMCPUCC pVCpu)
2539{
2540 /*
2541 * Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2542 * Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2543 *
2544 * Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2545 * cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2546 */
2547 PSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2548 bool const fWasCached = pVmcbNstGstCache->fCacheValid;
2549 if (!fWasCached)
2550 {
2551 PCSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2552 PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2553 pVmcbNstGstCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2554 pVmcbNstGstCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2555 pVmcbNstGstCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2556 pVmcbNstGstCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2557 pVmcbNstGstCache->u16PauseFilterThreshold = pVmcbNstGstCtrl->u16PauseFilterThreshold;
2558 pVmcbNstGstCache->u16PauseFilterCount = pVmcbNstGstCtrl->u16PauseFilterCount;
2559 pVmcbNstGstCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2560 pVmcbNstGstCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2561 pVmcbNstGstCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2562 pVmcbNstGstCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2563 pVmcbNstGstCache->fNestedPaging = pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging;
2564 pVmcbNstGstCache->fLbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2565 pVmcbNstGstCache->fCacheValid = true;
2566 Log4Func(("Cached VMCB fields\n"));
2567 }
2568
2569 return fWasCached;
2570}
2571
2572
2573/**
2574 * Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2575 *
2576 * This is done the first time we enter nested-guest execution using SVM R0
2577 * until the nested-guest \#VMEXIT (not to be confused with physical CPU
2578 * \#VMEXITs which may or may not cause a corresponding nested-guest \#VMEXIT).
2579 *
2580 * @param pVCpu The cross context virtual CPU structure.
2581 */
2582static void hmR0SvmSetupVmcbNested(PVMCPUCC pVCpu)
2583{
2584 PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
2585 PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2586
2587 HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
2588
2589 /*
2590 * First cache the nested-guest VMCB fields we may potentially modify.
2591 */
2592 bool const fVmcbCached = hmR0SvmCacheVmcbNested(pVCpu);
2593 if (!fVmcbCached)
2594 {
2595 /*
2596 * The IOPM of the nested-guest can be ignored because the the guest always
2597 * intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2598 * than the nested-guest IOPM and swap the field back on the #VMEXIT.
2599 */
2600 pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2601
2602 /*
2603 * Use the same nested-paging as the outer guest. We can't dynamically switch off
2604 * nested-paging suddenly while executing a VM (see assertion at the end of
2605 * Trap0eHandler() in PGMAllBth.h).
2606 */
2607 pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging;
2608
2609 /* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
2610 pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking = 1;
2611
2612 /*
2613 * Turn off TPR syncing on #VMEXIT for nested-guests as CR8 intercepts are subject
2614 * to the nested-guest intercepts and we always run with V_INTR_MASKING.
2615 */
2616 pVCpu->hmr0.s.svm.fSyncVTpr = false;
2617
2618# ifdef DEBUG_ramshankar
2619 /* For debugging purposes - copy the LBR info. from outer guest VMCB. */
2620 pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2621# endif
2622
2623 /*
2624 * If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
2625 * need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
2626 */
2627 if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
2628 pVmcbNstGstCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_VMSAVE
2629 | SVM_CTRL_INTERCEPT_VMLOAD;
2630
2631 /*
2632 * If we don't expose Virtual GIF feature to the outer guest, we need to intercept
2633 * CLGI/STGI instructions executed by the nested-guest.
2634 */
2635 if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVGif)
2636 pVmcbNstGstCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_CLGI
2637 | SVM_CTRL_INTERCEPT_STGI;
2638
2639 /* Merge the guest and nested-guest intercepts. */
2640 hmR0SvmMergeVmcbCtrlsNested(pVCpu);
2641
2642 /* Update the VMCB clean bits. */
2643 pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2644 }
2645 else
2646 {
2647 Assert(!pVCpu->hmr0.s.svm.fSyncVTpr);
2648 Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2649 Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPagingCtrl.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2650 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPagingCfg == pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
2651 }
2652}
2653
2654#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
2655
2656/**
2657 * Exports the state shared between the host and guest (or nested-guest) into
2658 * the VMCB.
2659 *
2660 * @param pVCpu The cross context virtual CPU structure.
2661 * @param pVmcb Pointer to the VM control block.
2662 *
2663 * @remarks No-long-jump zone!!!
2664 */
2665static void hmR0SvmExportSharedState(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
2666{
2667 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2668 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2669
2670 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
2671 hmR0SvmExportSharedDebugState(pVCpu, pVmcb);
2672
2673 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
2674 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE),
2675 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
2676}
2677
2678
2679/**
2680 * Worker for SVMR0ImportStateOnDemand.
2681 *
2682 * @param pVCpu The cross context virtual CPU structure.
2683 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2684 */
2685static void hmR0SvmImportGuestState(PVMCPUCC pVCpu, uint64_t fWhat)
2686{
2687 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
2688
2689 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
2690 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
2691 PCSVMVMCBSTATESAVE pVmcbGuest = &pVmcb->guest;
2692 PCSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
2693
2694 /*
2695 * We disable interrupts to make the updating of the state and in particular
2696 * the fExtrn modification atomic wrt to preemption hooks.
2697 */
2698 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
2699
2700 fWhat &= pCtx->fExtrn;
2701 if (fWhat)
2702 {
2703#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
2704 if (fWhat & CPUMCTX_EXTRN_HWVIRT)
2705 {
2706 if (pVmcbCtrl->IntCtrl.n.u1VGifEnable)
2707 {
2708 Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx)); /* We don't yet support passing VGIF feature to the guest. */
2709 Assert(HMIsSvmVGifActive(pVCpu->CTX_SUFF(pVM))); /* VM has configured it. */
2710 CPUMSetGuestGif(pCtx, pVmcbCtrl->IntCtrl.n.u1VGif);
2711 }
2712 }
2713
2714 if (fWhat & CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ)
2715 {
2716 if ( !pVmcbCtrl->IntCtrl.n.u1VIrqPending
2717 && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST))
2718 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST);
2719 }
2720#endif
2721
2722 if (fWhat & CPUMCTX_EXTRN_INHIBIT_INT)
2723 {
2724 if (pVmcbCtrl->IntShadow.n.u1IntShadow)
2725 EMSetInhibitInterruptsPC(pVCpu, pVmcbGuest->u64RIP);
2726 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2727 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2728 }
2729
2730 if (fWhat & CPUMCTX_EXTRN_RIP)
2731 pCtx->rip = pVmcbGuest->u64RIP;
2732
2733 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
2734 pCtx->eflags.u32 = pVmcbGuest->u64RFlags;
2735
2736 if (fWhat & CPUMCTX_EXTRN_RSP)
2737 pCtx->rsp = pVmcbGuest->u64RSP;
2738
2739 if (fWhat & CPUMCTX_EXTRN_RAX)
2740 pCtx->rax = pVmcbGuest->u64RAX;
2741
2742 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
2743 {
2744 if (fWhat & CPUMCTX_EXTRN_CS)
2745 {
2746 HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, CS, cs);
2747 /* Correct the CS granularity bit. Haven't seen it being wrong in any other register (yet). */
2748 /** @todo SELM might need to be fixed as it too should not care about the
2749 * granularity bit. See @bugref{6785}. */
2750 if ( !pCtx->cs.Attr.n.u1Granularity
2751 && pCtx->cs.Attr.n.u1Present
2752 && pCtx->cs.u32Limit > UINT32_C(0xfffff))
2753 {
2754 Assert((pCtx->cs.u32Limit & 0xfff) == 0xfff);
2755 pCtx->cs.Attr.n.u1Granularity = 1;
2756 }
2757 HMSVM_ASSERT_SEG_GRANULARITY(pCtx, cs);
2758 }
2759 if (fWhat & CPUMCTX_EXTRN_SS)
2760 {
2761 HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, SS, ss);
2762 HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ss);
2763 /*
2764 * Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the
2765 * VMCB and uses that and thus it's possible that when the CPL changes during
2766 * guest execution that the SS DPL isn't updated by AMD-V. Observed on some
2767 * AMD Fusion CPUs with 64-bit guests.
2768 *
2769 * See AMD spec. 15.5.1 "Basic operation".
2770 */
2771 Assert(!(pVmcbGuest->u8CPL & ~0x3));
2772 uint8_t const uCpl = pVmcbGuest->u8CPL;
2773 if (pCtx->ss.Attr.n.u2Dpl != uCpl)
2774 pCtx->ss.Attr.n.u2Dpl = uCpl & 0x3;
2775 }
2776 if (fWhat & CPUMCTX_EXTRN_DS)
2777 {
2778 HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, DS, ds);
2779 HMSVM_ASSERT_SEG_GRANULARITY(pCtx, ds);
2780 }
2781 if (fWhat & CPUMCTX_EXTRN_ES)
2782 {
2783 HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, ES, es);
2784 HMSVM_ASSERT_SEG_GRANULARITY(pCtx, es);
2785 }
2786 if (fWhat & CPUMCTX_EXTRN_FS)
2787 {
2788 HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, FS, fs);
2789 HMSVM_ASSERT_SEG_GRANULARITY(pCtx, fs);
2790 }
2791 if (fWhat & CPUMCTX_EXTRN_GS)
2792 {
2793 HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, GS, gs);
2794 HMSVM_ASSERT_SEG_GRANULARITY(pCtx, gs);
2795 }
2796 }
2797
2798 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
2799 {
2800 if (fWhat & CPUMCTX_EXTRN_TR)
2801 {
2802 /*
2803 * Fixup TR attributes so it's compatible with Intel. Important when saved-states
2804 * are used between Intel and AMD, see @bugref{6208#c39}.
2805 * ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2806 */
2807 HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, TR, tr);
2808 if (pCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2809 {
2810 if ( pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2811 || CPUMIsGuestInLongModeEx(pCtx))
2812 pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2813 else if (pCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2814 pCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2815 }
2816 }
2817
2818 if (fWhat & CPUMCTX_EXTRN_LDTR)
2819 HMSVM_SEG_REG_COPY_FROM_VMCB(pCtx, pVmcbGuest, LDTR, ldtr);
2820
2821 if (fWhat & CPUMCTX_EXTRN_GDTR)
2822 {
2823 pCtx->gdtr.cbGdt = pVmcbGuest->GDTR.u32Limit;
2824 pCtx->gdtr.pGdt = pVmcbGuest->GDTR.u64Base;
2825 }
2826
2827 if (fWhat & CPUMCTX_EXTRN_IDTR)
2828 {
2829 pCtx->idtr.cbIdt = pVmcbGuest->IDTR.u32Limit;
2830 pCtx->idtr.pIdt = pVmcbGuest->IDTR.u64Base;
2831 }
2832 }
2833
2834 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
2835 {
2836 pCtx->msrSTAR = pVmcbGuest->u64STAR;
2837 pCtx->msrLSTAR = pVmcbGuest->u64LSTAR;
2838 pCtx->msrCSTAR = pVmcbGuest->u64CSTAR;
2839 pCtx->msrSFMASK = pVmcbGuest->u64SFMASK;
2840 }
2841
2842 if ( (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
2843 && !pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit /* Intercepted. AMD-V would clear the high 32 bits of EIP & ESP. */)
2844 {
2845 pCtx->SysEnter.cs = pVmcbGuest->u64SysEnterCS;
2846 pCtx->SysEnter.eip = pVmcbGuest->u64SysEnterEIP;
2847 pCtx->SysEnter.esp = pVmcbGuest->u64SysEnterESP;
2848 }
2849
2850 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
2851 pCtx->msrKERNELGSBASE = pVmcbGuest->u64KernelGSBase;
2852
2853 if (fWhat & CPUMCTX_EXTRN_DR_MASK)
2854 {
2855 if (fWhat & CPUMCTX_EXTRN_DR6)
2856 {
2857 if (!pVCpu->hmr0.s.fUsingHyperDR7)
2858 pCtx->dr[6] = pVmcbGuest->u64DR6;
2859 else
2860 CPUMSetHyperDR6(pVCpu, pVmcbGuest->u64DR6);
2861 }
2862
2863 if (fWhat & CPUMCTX_EXTRN_DR7)
2864 {
2865 if (!pVCpu->hmr0.s.fUsingHyperDR7)
2866 pCtx->dr[7] = pVmcbGuest->u64DR7;
2867 else
2868 Assert(pVmcbGuest->u64DR7 == CPUMGetHyperDR7(pVCpu));
2869 }
2870 }
2871
2872 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
2873 {
2874 if (fWhat & CPUMCTX_EXTRN_CR0)
2875 {
2876 /* We intercept changes to all CR0 bits except maybe TS & MP bits. */
2877 uint64_t const uCr0 = (pCtx->cr0 & ~(X86_CR0_TS | X86_CR0_MP))
2878 | (pVmcbGuest->u64CR0 & (X86_CR0_TS | X86_CR0_MP));
2879 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
2880 CPUMSetGuestCR0(pVCpu, uCr0);
2881 VMMRZCallRing3Enable(pVCpu);
2882 }
2883
2884 if (fWhat & CPUMCTX_EXTRN_CR2)
2885 pCtx->cr2 = pVmcbGuest->u64CR2;
2886
2887 if (fWhat & CPUMCTX_EXTRN_CR3)
2888 {
2889 if ( pVmcbCtrl->NestedPagingCtrl.n.u1NestedPaging
2890 && pCtx->cr3 != pVmcbGuest->u64CR3)
2891 {
2892 CPUMSetGuestCR3(pVCpu, pVmcbGuest->u64CR3);
2893 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2894 }
2895 }
2896
2897 /* Changes to CR4 are always intercepted. */
2898 }
2899
2900 /* Update fExtrn. */
2901 pCtx->fExtrn &= ~fWhat;
2902
2903 /* If everything has been imported, clear the HM keeper bit. */
2904 if (!(pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL))
2905 {
2906 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
2907 Assert(!pCtx->fExtrn);
2908 }
2909 }
2910 else
2911 Assert(!pCtx->fExtrn || (pCtx->fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
2912
2913 ASMSetFlags(fEFlags);
2914
2915 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
2916
2917 /*
2918 * Honor any pending CR3 updates.
2919 *
2920 * Consider this scenario: #VMEXIT -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp
2921 * -> SVMR0CallRing3Callback() -> VMMRZCallRing3Disable() -> hmR0SvmImportGuestState()
2922 * -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp -> continue with #VMEXIT
2923 * handling -> hmR0SvmImportGuestState() and here we are.
2924 *
2925 * The reason for such complicated handling is because VM-exits that call into PGM expect
2926 * CR3 to be up-to-date and thus any CR3-saves -before- the VM-exit (longjmp) would've
2927 * postponed the CR3 update via the force-flag and cleared CR3 from fExtrn. Any SVM R0
2928 * VM-exit handler that requests CR3 to be saved will end up here and we call PGMUpdateCR3().
2929 *
2930 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again,
2931 * and does not process force-flag like regular exits to ring-3 either, we cover for it here.
2932 */
2933 if ( VMMRZCallRing3IsEnabled(pVCpu)
2934 && VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
2935 {
2936 AssertMsg(pCtx->cr3 == pVmcbGuest->u64CR3, ("cr3=%#RX64 vmcb_cr3=%#RX64\n", pCtx->cr3, pVmcbGuest->u64CR3));
2937 PGMUpdateCR3(pVCpu, pCtx->cr3);
2938 }
2939}
2940
2941
2942/**
2943 * Saves the guest (or nested-guest) state from the VMCB into the guest-CPU
2944 * context.
2945 *
2946 * Currently there is no residual state left in the CPU that is not updated in the
2947 * VMCB.
2948 *
2949 * @returns VBox status code.
2950 * @param pVCpu The cross context virtual CPU structure.
2951 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
2952 */
2953VMMR0DECL(int) SVMR0ImportStateOnDemand(PVMCPUCC pVCpu, uint64_t fWhat)
2954{
2955 hmR0SvmImportGuestState(pVCpu, fWhat);
2956 return VINF_SUCCESS;
2957}
2958
2959
2960/**
2961 * Does the necessary state syncing before returning to ring-3 for any reason
2962 * (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2963 *
2964 * @param pVCpu The cross context virtual CPU structure.
2965 * @param fImportState Whether to import the guest state from the VMCB back
2966 * to the guest-CPU context.
2967 *
2968 * @remarks No-long-jmp zone!!!
2969 */
2970static void hmR0SvmLeave(PVMCPUCC pVCpu, bool fImportState)
2971{
2972 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2973 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2974
2975 /*
2976 * !!! IMPORTANT !!!
2977 * If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
2978 */
2979
2980 /* Save the guest state if necessary. */
2981 if (fImportState)
2982 hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
2983
2984 /* Restore host FPU state if necessary and resync on next R0 reentry. */
2985 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2986 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
2987
2988 /*
2989 * Restore host debug registers if necessary and resync on next R0 reentry.
2990 */
2991#ifdef VBOX_STRICT
2992 if (CPUMIsHyperDebugStateActive(pVCpu))
2993 {
2994 PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb; /** @todo nested-guest. */
2995 Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2996 Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2997 }
2998#endif
2999 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3000 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3001 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3002
3003 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
3004 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
3005 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
3006 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
3007 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
3008 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitVmentry);
3009 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3010
3011 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
3012}
3013
3014
3015/**
3016 * Leaves the AMD-V session.
3017 *
3018 * Only used while returning to ring-3 either due to longjump or exits to
3019 * ring-3.
3020 *
3021 * @returns VBox status code.
3022 * @param pVCpu The cross context virtual CPU structure.
3023 */
3024static int hmR0SvmLeaveSession(PVMCPUCC pVCpu)
3025{
3026 HM_DISABLE_PREEMPT(pVCpu);
3027 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3028 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3029
3030 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
3031 and done this from the SVMR0ThreadCtxCallback(). */
3032 if (!pVCpu->hmr0.s.fLeaveDone)
3033 {
3034 hmR0SvmLeave(pVCpu, true /* fImportState */);
3035 pVCpu->hmr0.s.fLeaveDone = true;
3036 }
3037
3038 /*
3039 * !!! IMPORTANT !!!
3040 * If you modify code here, make sure to check whether SVMR0CallRing3Callback() needs to be updated too.
3041 */
3042
3043 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3044 /* Deregister hook now that we've left HM context before re-enabling preemption. */
3045 VMMR0ThreadCtxHookDisable(pVCpu);
3046
3047 /* Leave HM context. This takes care of local init (term). */
3048 int rc = HMR0LeaveCpu(pVCpu);
3049
3050 HM_RESTORE_PREEMPT();
3051 return rc;
3052}
3053
3054
3055/**
3056 * VMMRZCallRing3() callback wrapper which saves the guest state (or restores
3057 * any remaining host state) before we go back to ring-3 due to an assertion.
3058 *
3059 * @param pVCpu The cross context virtual CPU structure.
3060 */
3061VMMR0DECL(int) SVMR0AssertionCallback(PVMCPUCC pVCpu)
3062{
3063 /*
3064 * !!! IMPORTANT !!!
3065 * If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
3066 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
3067 */
3068 VMMR0AssertionRemoveNotification(pVCpu);
3069 VMMRZCallRing3Disable(pVCpu);
3070 HM_DISABLE_PREEMPT(pVCpu);
3071
3072 /* Import the entire guest state. */
3073 hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3074
3075 /* Restore host FPU state if necessary and resync on next R0 reentry. */
3076 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
3077
3078 /* Restore host debug registers if necessary and resync on next R0 reentry. */
3079 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
3080
3081 /* Deregister the hook now that we've left HM context before re-enabling preemption. */
3082 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
3083 VMMR0ThreadCtxHookDisable(pVCpu);
3084
3085 /* Leave HM context. This takes care of local init (term). */
3086 HMR0LeaveCpu(pVCpu);
3087
3088 HM_RESTORE_PREEMPT();
3089 return VINF_SUCCESS;
3090}
3091
3092
3093/**
3094 * Take necessary actions before going back to ring-3.
3095 *
3096 * An action requires us to go back to ring-3. This function does the necessary
3097 * steps before we can safely return to ring-3. This is not the same as longjmps
3098 * to ring-3, this is voluntary.
3099 *
3100 * @returns Strict VBox status code.
3101 * @param pVCpu The cross context virtual CPU structure.
3102 * @param rcExit The reason for exiting to ring-3. Can be
3103 * VINF_VMM_UNKNOWN_RING3_CALL.
3104 */
3105static VBOXSTRICTRC hmR0SvmExitToRing3(PVMCPUCC pVCpu, VBOXSTRICTRC rcExit)
3106{
3107 Assert(pVCpu);
3108 HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3109
3110 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
3111 VMMRZCallRing3Disable(pVCpu);
3112 Log4Func(("rcExit=%d LocalFF=%#RX64 GlobalFF=%#RX32\n", VBOXSTRICTRC_VAL(rcExit), (uint64_t)pVCpu->fLocalForcedActions,
3113 pVCpu->CTX_SUFF(pVM)->fGlobalForcedActions));
3114
3115 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
3116 if (pVCpu->hm.s.Event.fPending)
3117 {
3118 hmR0SvmPendingEventToTrpmTrap(pVCpu);
3119 Assert(!pVCpu->hm.s.Event.fPending);
3120 }
3121
3122 /* Sync. the necessary state for going back to ring-3. */
3123 hmR0SvmLeaveSession(pVCpu);
3124 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
3125
3126 /* Thread-context hooks are unregistered at this point!!! */
3127 /* Ring-3 callback notifications are unregistered at this point!!! */
3128
3129 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
3130 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
3131 | CPUM_CHANGED_LDTR
3132 | CPUM_CHANGED_GDTR
3133 | CPUM_CHANGED_IDTR
3134 | CPUM_CHANGED_TR
3135 | CPUM_CHANGED_HIDDEN_SEL_REGS);
3136 if ( pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging
3137 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
3138 {
3139 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
3140 }
3141
3142 /* Update the exit-to-ring 3 reason. */
3143 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
3144
3145 /* On our way back from ring-3, reload the guest-CPU state if it may change while in ring-3. */
3146 if ( rcExit != VINF_EM_RAW_INTERRUPT
3147 || CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3148 {
3149 Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
3150 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
3151 }
3152
3153 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
3154 VMMRZCallRing3Enable(pVCpu);
3155
3156 /*
3157 * If we're emulating an instruction, we shouldn't have any TRPM traps pending
3158 * and if we're injecting an event we should have a TRPM trap pending.
3159 */
3160 AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu),
3161 pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3162 VERR_SVM_IPE_5);
3163 AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu),
3164 pVCpu->hm.s.u32HMError = VBOXSTRICTRC_VAL(rcExit),
3165 VERR_SVM_IPE_4);
3166
3167 return rcExit;
3168}
3169
3170
3171/**
3172 * Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
3173 * intercepts.
3174 *
3175 * @param pVCpu The cross context virtual CPU structure.
3176 * @param pVmcb Pointer to the VM control block.
3177 *
3178 * @remarks No-long-jump zone!!!
3179 */
3180static void hmR0SvmUpdateTscOffsetting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3181{
3182 /*
3183 * Avoid intercepting RDTSC/RDTSCP if we determined the host TSC (++) is stable
3184 * and in case of a nested-guest, if the nested-VMCB specifies it is not intercepting
3185 * RDTSC/RDTSCP as well.
3186 */
3187 bool fParavirtTsc;
3188 uint64_t uTscOffset;
3189 bool const fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVCpu->CTX_SUFF(pVM), pVCpu, &uTscOffset, &fParavirtTsc);
3190
3191 bool fIntercept;
3192 if (fCanUseRealTsc)
3193 fIntercept = hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_RDTSC | SVM_CTRL_INTERCEPT_RDTSCP);
3194 else
3195 {
3196 hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_RDTSC | SVM_CTRL_INTERCEPT_RDTSCP);
3197 fIntercept = true;
3198 }
3199
3200 if (!fIntercept)
3201 {
3202#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3203 /* Apply the nested-guest VMCB's TSC offset over the guest TSC offset. */
3204 if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
3205 uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
3206#endif
3207
3208 /* Update the TSC offset in the VMCB and the relevant clean bits. */
3209 pVmcb->ctrl.u64TSCOffset = uTscOffset;
3210 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3211 }
3212
3213 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
3214 information before every VM-entry, hence we have nothing to do here at the moment. */
3215 if (fParavirtTsc)
3216 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
3217}
3218
3219
3220/**
3221 * Sets an event as a pending event to be injected into the guest.
3222 *
3223 * @param pVCpu The cross context virtual CPU structure.
3224 * @param pEvent Pointer to the SVM event.
3225 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
3226 * page-fault.
3227 *
3228 * @remarks Statistics counter assumes this is a guest event being reflected to
3229 * the guest i.e. 'StatInjectPendingReflect' is incremented always.
3230 */
3231DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPUCC pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
3232{
3233 Assert(!pVCpu->hm.s.Event.fPending);
3234 Assert(pEvent->n.u1Valid);
3235
3236 pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
3237 pVCpu->hm.s.Event.fPending = true;
3238 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
3239
3240 Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3241 (uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3242}
3243
3244
3245/**
3246 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
3247 *
3248 * @param pVCpu The cross context virtual CPU structure.
3249 */
3250DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPUCC pVCpu)
3251{
3252 SVMEVENT Event;
3253 Event.u = 0;
3254 Event.n.u1Valid = 1;
3255 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3256 Event.n.u8Vector = X86_XCPT_UD;
3257 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3258}
3259
3260
3261/**
3262 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
3263 *
3264 * @param pVCpu The cross context virtual CPU structure.
3265 */
3266DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPUCC pVCpu)
3267{
3268 SVMEVENT Event;
3269 Event.u = 0;
3270 Event.n.u1Valid = 1;
3271 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3272 Event.n.u8Vector = X86_XCPT_DB;
3273 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3274}
3275
3276
3277/**
3278 * Sets a page fault (\#PF) exception as pending-for-injection into the VM.
3279 *
3280 * @param pVCpu The cross context virtual CPU structure.
3281 * @param u32ErrCode The error-code for the page-fault.
3282 * @param uFaultAddress The page fault address (CR2).
3283 *
3284 * @remarks This updates the guest CR2 with @a uFaultAddress!
3285 */
3286DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPUCC pVCpu, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
3287{
3288 SVMEVENT Event;
3289 Event.u = 0;
3290 Event.n.u1Valid = 1;
3291 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3292 Event.n.u8Vector = X86_XCPT_PF;
3293 Event.n.u1ErrorCodeValid = 1;
3294 Event.n.u32ErrorCode = u32ErrCode;
3295
3296 /* Update CR2 of the guest. */
3297 HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR2);
3298 if (pVCpu->cpum.GstCtx.cr2 != uFaultAddress)
3299 {
3300 pVCpu->cpum.GstCtx.cr2 = uFaultAddress;
3301 /* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
3302 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
3303 }
3304
3305 hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
3306}
3307
3308
3309/**
3310 * Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
3311 *
3312 * @param pVCpu The cross context virtual CPU structure.
3313 */
3314DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPUCC pVCpu)
3315{
3316 SVMEVENT Event;
3317 Event.u = 0;
3318 Event.n.u1Valid = 1;
3319 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3320 Event.n.u8Vector = X86_XCPT_MF;
3321 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3322}
3323
3324
3325/**
3326 * Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3327 *
3328 * @param pVCpu The cross context virtual CPU structure.
3329 */
3330DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPUCC pVCpu)
3331{
3332 SVMEVENT Event;
3333 Event.u = 0;
3334 Event.n.u1Valid = 1;
3335 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3336 Event.n.u8Vector = X86_XCPT_DF;
3337 Event.n.u1ErrorCodeValid = 1;
3338 Event.n.u32ErrorCode = 0;
3339 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3340}
3341
3342
3343/**
3344 * Injects an event into the guest upon VMRUN by updating the relevant field
3345 * in the VMCB.
3346 *
3347 * @param pVCpu The cross context virtual CPU structure.
3348 * @param pVmcb Pointer to the guest VM control block.
3349 * @param pEvent Pointer to the event.
3350 *
3351 * @remarks No-long-jump zone!!!
3352 * @remarks Requires CR0!
3353 */
3354DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMEVENT pEvent)
3355{
3356 Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3357 pVmcb->ctrl.EventInject.u = pEvent->u;
3358 if ( pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_EXCEPTION
3359 || pVmcb->ctrl.EventInject.n.u3Type == SVM_EVENT_NMI)
3360 {
3361 Assert(pEvent->n.u8Vector <= X86_XCPT_LAST);
3362 STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedXcpts[pEvent->n.u8Vector]);
3363 }
3364 else
3365 STAM_COUNTER_INC(&pVCpu->hm.s.aStatInjectedIrqs[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3366 RT_NOREF(pVCpu);
3367
3368 Log4Func(("u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u, pEvent->n.u8Vector,
3369 (uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3370}
3371
3372
3373
3374/**
3375 * Converts any TRPM trap into a pending HM event. This is typically used when
3376 * entering from ring-3 (not longjmp returns).
3377 *
3378 * @param pVCpu The cross context virtual CPU structure.
3379 */
3380static void hmR0SvmTrpmTrapToPendingEvent(PVMCPUCC pVCpu)
3381{
3382 Assert(TRPMHasTrap(pVCpu));
3383 Assert(!pVCpu->hm.s.Event.fPending);
3384
3385 uint8_t uVector;
3386 TRPMEVENT enmTrpmEvent;
3387 uint32_t uErrCode;
3388 RTGCUINTPTR GCPtrFaultAddress;
3389 uint8_t cbInstr;
3390
3391 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr, NULL /* pfIcebp */);
3392 AssertRC(rc);
3393
3394 SVMEVENT Event;
3395 Event.u = 0;
3396 Event.n.u1Valid = 1;
3397 Event.n.u8Vector = uVector;
3398
3399 /* Refer AMD spec. 15.20 "Event Injection" for the format. */
3400 if (enmTrpmEvent == TRPM_TRAP)
3401 {
3402 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3403 switch (uVector)
3404 {
3405 case X86_XCPT_NMI:
3406 {
3407 Event.n.u3Type = SVM_EVENT_NMI;
3408 break;
3409 }
3410
3411 case X86_XCPT_BP:
3412 case X86_XCPT_OF:
3413 AssertMsgFailed(("Invalid TRPM vector %d for event type %d\n", uVector, enmTrpmEvent));
3414 RT_FALL_THRU();
3415
3416 case X86_XCPT_PF:
3417 case X86_XCPT_DF:
3418 case X86_XCPT_TS:
3419 case X86_XCPT_NP:
3420 case X86_XCPT_SS:
3421 case X86_XCPT_GP:
3422 case X86_XCPT_AC:
3423 {
3424 Event.n.u1ErrorCodeValid = 1;
3425 Event.n.u32ErrorCode = uErrCode;
3426 break;
3427 }
3428 }
3429 }
3430 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3431 Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3432 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3433 Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3434 else
3435 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3436
3437 rc = TRPMResetTrap(pVCpu);
3438 AssertRC(rc);
3439
3440 Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3441 !!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3442
3443 hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3444}
3445
3446
3447/**
3448 * Converts any pending SVM event into a TRPM trap. Typically used when leaving
3449 * AMD-V to execute any instruction.
3450 *
3451 * @param pVCpu The cross context virtual CPU structure.
3452 */
3453static void hmR0SvmPendingEventToTrpmTrap(PVMCPUCC pVCpu)
3454{
3455 Assert(pVCpu->hm.s.Event.fPending);
3456 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3457
3458 SVMEVENT Event;
3459 Event.u = pVCpu->hm.s.Event.u64IntInfo;
3460
3461 uint8_t uVector = Event.n.u8Vector;
3462 TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event, uVector);
3463
3464 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, Event.n.u3Type));
3465
3466 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3467 AssertRC(rc);
3468
3469 if (Event.n.u1ErrorCodeValid)
3470 TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3471
3472 if ( enmTrapType == TRPM_TRAP
3473 && uVector == X86_XCPT_PF)
3474 {
3475 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3476 Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3477 }
3478 else if (enmTrapType == TRPM_SOFTWARE_INT)
3479 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3480 pVCpu->hm.s.Event.fPending = false;
3481}
3482
3483
3484/**
3485 * Checks if the guest (or nested-guest) has an interrupt shadow active right
3486 * now.
3487 *
3488 * @returns @c true if the interrupt shadow is active, @c false otherwise.
3489 * @param pVCpu The cross context virtual CPU structure.
3490 *
3491 * @remarks No-long-jump zone!!!
3492 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3493 */
3494static bool hmR0SvmIsIntrShadowActive(PVMCPUCC pVCpu)
3495{
3496 /*
3497 * Instructions like STI and MOV SS inhibit interrupts till the next instruction
3498 * completes. Check if we should inhibit interrupts or clear any existing
3499 * interrupt inhibition.
3500 */
3501 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3502 {
3503 if (pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
3504 {
3505 /*
3506 * We can clear the inhibit force flag as even if we go back to the recompiler
3507 * without executing guest code in AMD-V, the flag's condition to be cleared is
3508 * met and thus the cleared state is correct.
3509 */
3510 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3511 return false;
3512 }
3513 return true;
3514 }
3515 return false;
3516}
3517
3518
3519/**
3520 * Sets the virtual interrupt intercept control in the VMCB.
3521 *
3522 * @param pVCpu The cross context virtual CPU structure.
3523 * @param pVmcb Pointer to the VM control block.
3524 */
3525static void hmR0SvmSetIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3526{
3527 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3528
3529 /*
3530 * When AVIC isn't supported, set up an interrupt window to cause a #VMEXIT when the guest
3531 * is ready to accept interrupts. At #VMEXIT, we then get the interrupt from the APIC
3532 * (updating ISR at the right time) and inject the interrupt.
3533 *
3534 * With AVIC is supported, we could make use of the asynchronously delivery without
3535 * #VMEXIT and we would be passing the AVIC page to SVM.
3536 *
3537 * In AMD-V, an interrupt window is achieved using a combination of V_IRQ (an interrupt
3538 * is pending), V_IGN_TPR (ignore TPR priorities) and the VINTR intercept all being set.
3539 */
3540 Assert(pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR);
3541 pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3542 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3543 hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3544 Log4(("Set VINTR intercept\n"));
3545}
3546
3547
3548/**
3549 * Clears the virtual interrupt intercept control in the VMCB as
3550 * we are figured the guest is unable process any interrupts
3551 * at this point of time.
3552 *
3553 * @param pVCpu The cross context virtual CPU structure.
3554 * @param pVmcb Pointer to the VM control block.
3555 */
3556static void hmR0SvmClearIntWindowExiting(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3557{
3558 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx); NOREF(pVCpu);
3559
3560 PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
3561 if ( pVmcbCtrl->IntCtrl.n.u1VIrqPending
3562 || (pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3563 {
3564 pVmcbCtrl->IntCtrl.n.u1VIrqPending = 0;
3565 pVmcbCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INT_CTRL;
3566 hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_VINTR);
3567 Log4(("Cleared VINTR intercept\n"));
3568 }
3569}
3570
3571
3572/**
3573 * Evaluates the event to be delivered to the guest and sets it as the pending
3574 * event.
3575 *
3576 * @returns Strict VBox status code.
3577 * @param pVCpu The cross context virtual CPU structure.
3578 * @param pSvmTransient Pointer to the SVM transient structure.
3579 */
3580static VBOXSTRICTRC hmR0SvmEvaluatePendingEvent(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient)
3581{
3582 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3583 HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_HWVIRT
3584 | CPUMCTX_EXTRN_RFLAGS
3585 | CPUMCTX_EXTRN_INHIBIT_INT
3586 | CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ);
3587
3588 Assert(!pVCpu->hm.s.Event.fPending);
3589 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3590 Assert(pVmcb);
3591
3592 bool const fGif = CPUMGetGuestGif(pCtx);
3593 bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3594 bool const fBlockNmi = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3595
3596 Log4Func(("fGif=%RTbool fBlockNmi=%RTbool fIntShadow=%RTbool fIntPending=%RTbool fNmiPending=%RTbool\n",
3597 fGif, fBlockNmi, fIntShadow, VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC),
3598 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)));
3599
3600 /** @todo SMI. SMIs take priority over NMIs. */
3601
3602 /*
3603 * Check if the guest or nested-guest can receive NMIs.
3604 * Nested NMIs are not allowed, see AMD spec. 8.1.4 "Masking External Interrupts".
3605 * NMIs take priority over maskable interrupts, see AMD spec. 8.5 "Priorities".
3606 */
3607 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)
3608 && !fBlockNmi)
3609 {
3610 if ( fGif
3611 && !fIntShadow)
3612 {
3613#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3614 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_NMI))
3615 {
3616 Log4(("Intercepting NMI -> #VMEXIT\n"));
3617 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3618 return IEMExecSvmVmexit(pVCpu, SVM_EXIT_NMI, 0, 0);
3619 }
3620#endif
3621 Log4(("Setting NMI pending for injection\n"));
3622 SVMEVENT Event;
3623 Event.u = 0;
3624 Event.n.u1Valid = 1;
3625 Event.n.u8Vector = X86_XCPT_NMI;
3626 Event.n.u3Type = SVM_EVENT_NMI;
3627 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3628 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3629 }
3630 else if (!fGif)
3631 hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3632 else if (!pSvmTransient->fIsNestedGuest)
3633 hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3634 /* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3635 }
3636 /*
3637 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt()
3638 * returns a valid interrupt we -must- deliver the interrupt. We can no longer re-request
3639 * it from the APIC device.
3640 *
3641 * For nested-guests, physical interrupts always take priority over virtual interrupts.
3642 * We don't need to inject nested-guest virtual interrupts here, we can let the hardware
3643 * do that work when we execute nested-guest code esp. since all the required information
3644 * is in the VMCB, unlike physical interrupts where we need to fetch the interrupt from
3645 * the virtual interrupt controller.
3646 *
3647 * See AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3648 */
3649 else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
3650 && !pVCpu->hm.s.fSingleInstruction)
3651 {
3652 bool const fBlockInt = !pSvmTransient->fIsNestedGuest ? !(pCtx->eflags.u32 & X86_EFL_IF)
3653 : CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx);
3654 if ( fGif
3655 && !fBlockInt
3656 && !fIntShadow)
3657 {
3658#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
3659 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INTR))
3660 {
3661 Log4(("Intercepting INTR -> #VMEXIT\n"));
3662 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3663 return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3664 }
3665#endif
3666 uint8_t u8Interrupt;
3667 int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3668 if (RT_SUCCESS(rc))
3669 {
3670 Log4(("Setting external interrupt %#x pending for injection\n", u8Interrupt));
3671 SVMEVENT Event;
3672 Event.u = 0;
3673 Event.n.u1Valid = 1;
3674 Event.n.u8Vector = u8Interrupt;
3675 Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3676 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3677 }
3678 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3679 {
3680 /*
3681 * AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3682 * updated eventually when the TPR is written by the guest.
3683 */
3684 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3685 }
3686 else
3687 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3688 }
3689 else if (!fGif)
3690 hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_STGI);
3691 else if (!pSvmTransient->fIsNestedGuest)
3692 hmR0SvmSetIntWindowExiting(pVCpu, pVmcb);
3693 /* else: for nested-guests, interrupt-window exiting will be picked up when merging VMCB controls. */
3694 }
3695
3696 return VINF_SUCCESS;
3697}
3698
3699
3700/**
3701 * Injects any pending events into the guest (or nested-guest).
3702 *
3703 * @param pVCpu The cross context virtual CPU structure.
3704 * @param pVmcb Pointer to the VM control block.
3705 *
3706 * @remarks Must only be called when we are guaranteed to enter
3707 * hardware-assisted SVM execution and not return to ring-3
3708 * prematurely.
3709 */
3710static void hmR0SvmInjectPendingEvent(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
3711{
3712 Assert(!TRPMHasTrap(pVCpu));
3713 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3714
3715 bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu);
3716#ifdef VBOX_STRICT
3717 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3718 bool const fGif = CPUMGetGuestGif(pCtx);
3719 bool fAllowInt = fGif;
3720 if (fGif)
3721 {
3722 /*
3723 * For nested-guests we have no way to determine if we're injecting a physical or
3724 * virtual interrupt at this point. Hence the partial verification below.
3725 */
3726 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3727 fAllowInt = CPUMIsGuestSvmPhysIntrEnabled(pVCpu, pCtx) || CPUMIsGuestSvmVirtIntrEnabled(pVCpu, pCtx);
3728 else
3729 fAllowInt = RT_BOOL(pCtx->eflags.u32 & X86_EFL_IF);
3730 }
3731#endif
3732
3733 if (pVCpu->hm.s.Event.fPending)
3734 {
3735 SVMEVENT Event;
3736 Event.u = pVCpu->hm.s.Event.u64IntInfo;
3737 Assert(Event.n.u1Valid);
3738
3739 /*
3740 * Validate event injection pre-conditions.
3741 */
3742 if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3743 {
3744 Assert(fAllowInt);
3745 Assert(!fIntShadow);
3746 }
3747 else if (Event.n.u3Type == SVM_EVENT_NMI)
3748 {
3749 Assert(fGif);
3750 Assert(!fIntShadow);
3751 }
3752
3753 /*
3754 * Before injecting an NMI we must set VMCPU_FF_BLOCK_NMIS to prevent nested NMIs. We
3755 * do this only when we are surely going to inject the NMI as otherwise if we return
3756 * to ring-3 prematurely we could leave NMIs blocked indefinitely upon re-entry into
3757 * SVM R0.
3758 *
3759 * With VT-x, this is handled by the Guest interruptibility information VMCS field
3760 * which will set the VMCS field after actually delivering the NMI which we read on
3761 * VM-exit to determine the state.
3762 */
3763 if ( Event.n.u3Type == SVM_EVENT_NMI
3764 && Event.n.u8Vector == X86_XCPT_NMI
3765 && !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3766 {
3767 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3768 }
3769
3770 /*
3771 * Inject it (update VMCB for injection by the hardware).
3772 */
3773 Log4(("Injecting pending HM event\n"));
3774 hmR0SvmInjectEventVmcb(pVCpu, pVmcb, &Event);
3775 pVCpu->hm.s.Event.fPending = false;
3776
3777 if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3778 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3779 else
3780 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3781 }
3782 else
3783 Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3784
3785 /*
3786 * We could have injected an NMI through IEM and continue guest execution using
3787 * hardware-assisted SVM. In which case, we would not have any events pending (above)
3788 * but we still need to intercept IRET in order to eventually clear NMI inhibition.
3789 */
3790 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
3791 hmR0SvmSetCtrlIntercept(pVmcb, SVM_CTRL_INTERCEPT_IRET);
3792
3793 /*
3794 * Update the guest interrupt shadow in the guest (or nested-guest) VMCB.
3795 *
3796 * For nested-guests: We need to update it too for the scenario where IEM executes
3797 * the nested-guest but execution later continues here with an interrupt shadow active.
3798 */
3799 pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3800}
3801
3802
3803/**
3804 * Reports world-switch error and dumps some useful debug info.
3805 *
3806 * @param pVCpu The cross context virtual CPU structure.
3807 * @param rcVMRun The return code from VMRUN (or
3808 * VERR_SVM_INVALID_GUEST_STATE for invalid
3809 * guest-state).
3810 */
3811static void hmR0SvmReportWorldSwitchError(PVMCPUCC pVCpu, int rcVMRun)
3812{
3813 HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
3814 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
3815 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
3816
3817 if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3818 {
3819#ifdef VBOX_STRICT
3820 hmR0DumpRegs(pVCpu, HM_DUMP_REG_FLAGS_ALL);
3821 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
3822 Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3823 Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3824 Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3825 Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3826 Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3827 Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3828 Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3829 Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3830 Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3831 Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3832
3833 Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3834 Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3835 Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3836
3837 Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3838 Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3839 Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3840 Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved));
3841 Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3842 Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3843 Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3844 Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3845 Log4(("ctrl.IntCtrl.u1VGifEnable %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGifEnable));
3846 Log4(("ctrl.IntCtrl.u5Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u5Reserved));
3847 Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3848 Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3849
3850 Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3851 Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3852 Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3853 Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3854 Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3855 Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3856 Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3857 Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3858 Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3859 Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3860 Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3861 Log4(("ctrl.NestedPagingCtrl.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1NestedPaging));
3862 Log4(("ctrl.NestedPagingCtrl.u1Sev %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1Sev));
3863 Log4(("ctrl.NestedPagingCtrl.u1SevEs %#x\n", pVmcb->ctrl.NestedPagingCtrl.n.u1SevEs));
3864 Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3865 Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3866 Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3867 Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3868 Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3869 Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3870
3871 Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3872
3873 Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3874 Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3875
3876 Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3877 Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3878 Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3879 Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3880 Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3881 Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3882 Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3883 Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3884 Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3885 Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3886 Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3887 Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3888 Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3889 Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3890 Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3891 Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3892 Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3893 Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3894 Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3895 Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3896
3897 Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3898 Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3899
3900 Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3901 Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3902 Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3903 Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3904
3905 Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3906 Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3907
3908 Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3909 Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3910 Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3911 Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3912
3913 Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3914 Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3915 Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3916 Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3917 Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3918 Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3919 Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3920
3921 Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3922 Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3923 Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3924 Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3925
3926 Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3927 Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3928 Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3929
3930 Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3931 Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3932 Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3933 Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3934 Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3935 Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3936 Log4(("guest.u64PAT %#RX64\n", pVmcb->guest.u64PAT));
3937 Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3938 Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3939 Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3940 Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3941 Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3942
3943 NOREF(pVmcb);
3944#endif /* VBOX_STRICT */
3945 }
3946 else
3947 Log4Func(("rcVMRun=%d\n", rcVMRun));
3948}
3949
3950
3951/**
3952 * Check per-VM and per-VCPU force flag actions that require us to go back to
3953 * ring-3 for one reason or another.
3954 *
3955 * @returns Strict VBox status code (information status code included).
3956 * @retval VINF_SUCCESS if we don't have any actions that require going back to
3957 * ring-3.
3958 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3959 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3960 * interrupts)
3961 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3962 * all EMTs to be in ring-3.
3963 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3964 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3965 * to the EM loop.
3966 *
3967 * @param pVCpu The cross context virtual CPU structure.
3968 */
3969static VBOXSTRICTRC hmR0SvmCheckForceFlags(PVMCPUCC pVCpu)
3970{
3971 Assert(VMMRZCallRing3IsEnabled(pVCpu));
3972
3973 /* Could happen as a result of longjump. */
3974 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
3975 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
3976
3977 /* Update pending interrupts into the APIC's IRR. */
3978 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3979 APICUpdatePendingInterrupts(pVCpu);
3980
3981 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3982 if ( VM_FF_IS_ANY_SET(pVM, !pVCpu->hm.s.fSingleInstruction
3983 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3984 || VMCPU_FF_IS_ANY_SET(pVCpu, !pVCpu->hm.s.fSingleInstruction
3985 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3986 {
3987 /* Pending PGM C3 sync. */
3988 if (VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3989 {
3990 int rc = PGMSyncCR3(pVCpu, pVCpu->cpum.GstCtx.cr0, pVCpu->cpum.GstCtx.cr3, pVCpu->cpum.GstCtx.cr4,
3991 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3992 if (rc != VINF_SUCCESS)
3993 {
3994 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3995 return rc;
3996 }
3997 }
3998
3999 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
4000 /* -XXX- what was that about single stepping? */
4001 if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
4002 || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4003 {
4004 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4005 int rc = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
4006 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
4007 return rc;
4008 }
4009
4010 /* Pending VM request packets, such as hardware interrupts. */
4011 if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
4012 || VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
4013 {
4014 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchVmReq);
4015 Log4Func(("Pending VM request forcing us back to ring-3\n"));
4016 return VINF_EM_PENDING_REQUEST;
4017 }
4018
4019 /* Pending PGM pool flushes. */
4020 if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
4021 {
4022 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPgmPoolFlush);
4023 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
4024 return VINF_PGM_POOL_FLUSH_PENDING;
4025 }
4026
4027 /* Pending DMA requests. */
4028 if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
4029 {
4030 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchDma);
4031 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
4032 return VINF_EM_RAW_TO_R3;
4033 }
4034 }
4035
4036 return VINF_SUCCESS;
4037}
4038
4039
4040/**
4041 * Does the preparations before executing guest code in AMD-V.
4042 *
4043 * This may cause longjmps to ring-3 and may even result in rescheduling to the
4044 * recompiler. We must be cautious what we do here regarding committing
4045 * guest-state information into the VMCB assuming we assuredly execute the guest
4046 * in AMD-V. If we fall back to the recompiler after updating the VMCB and
4047 * clearing the common-state (TRPM/forceflags), we must undo those changes so
4048 * that the recompiler can (and should) use them when it resumes guest
4049 * execution. Otherwise such operations must be done when we can no longer
4050 * exit to ring-3.
4051 *
4052 * @returns Strict VBox status code (informational status codes included).
4053 * @retval VINF_SUCCESS if we can proceed with running the guest.
4054 * @retval VINF_* scheduling changes, we have to go back to ring-3.
4055 *
4056 * @param pVCpu The cross context virtual CPU structure.
4057 * @param pSvmTransient Pointer to the SVM transient structure.
4058 */
4059static VBOXSTRICTRC hmR0SvmPreRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4060{
4061 HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4062
4063#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
4064 if (pSvmTransient->fIsNestedGuest)
4065 {
4066 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
4067 return VINF_EM_RESCHEDULE_REM;
4068 }
4069#endif
4070
4071 /* Check force flag actions that might require us to go back to ring-3. */
4072 VBOXSTRICTRC rc = hmR0SvmCheckForceFlags(pVCpu);
4073 if (rc != VINF_SUCCESS)
4074 return rc;
4075
4076 if (TRPMHasTrap(pVCpu))
4077 hmR0SvmTrpmTrapToPendingEvent(pVCpu);
4078 else if (!pVCpu->hm.s.Event.fPending)
4079 {
4080 rc = hmR0SvmEvaluatePendingEvent(pVCpu, pSvmTransient);
4081 if ( rc != VINF_SUCCESS
4082 || pSvmTransient->fIsNestedGuest != CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4083 {
4084 /* If a nested-guest VM-exit occurred, bail. */
4085 if (pSvmTransient->fIsNestedGuest)
4086 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4087 return rc;
4088 }
4089 }
4090
4091 /*
4092 * On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
4093 * Just do it in software, see @bugref{8411}.
4094 * NB: If we could continue a task switch exit we wouldn't need to do this.
4095 */
4096 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4097 if (RT_UNLIKELY( !g_fHmSvmFeatures
4098 && pVCpu->hm.s.Event.fPending
4099 && SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
4100 return VINF_EM_RAW_INJECT_TRPM_EVENT;
4101
4102#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4103 Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4104 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
4105#endif
4106
4107#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4108 /*
4109 * Set up the nested-guest VMCB for execution using hardware-assisted SVM.
4110 */
4111 if (pSvmTransient->fIsNestedGuest)
4112 hmR0SvmSetupVmcbNested(pVCpu);
4113#endif
4114
4115 /*
4116 * Export the guest state bits that are not shared with the host in any way as we can
4117 * longjmp or get preempted in the midst of exporting some of the state.
4118 */
4119 rc = hmR0SvmExportGuestState(pVCpu, pSvmTransient);
4120 AssertRCReturn(rc, rc);
4121 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
4122
4123 /* Ensure we've cached (and hopefully modified) the nested-guest VMCB for execution using hardware-assisted SVM. */
4124 Assert(!pSvmTransient->fIsNestedGuest || pVCpu->hm.s.svm.NstGstVmcbCache.fCacheValid);
4125
4126 /*
4127 * If we're not intercepting TPR changes in the guest, save the guest TPR before the
4128 * world-switch so we can update it on the way back if the guest changed the TPR.
4129 */
4130 if (pVCpu->hmr0.s.svm.fSyncVTpr)
4131 {
4132 Assert(!pSvmTransient->fIsNestedGuest);
4133 PCSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4134 if (pVM->hm.s.fTprPatchingActive)
4135 pSvmTransient->u8GuestTpr = pVmcb->guest.u64LSTAR;
4136 else
4137 pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4138 }
4139
4140 /*
4141 * No longjmps to ring-3 from this point on!!!
4142 *
4143 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4144 * better than a kernel panic. This also disables flushing of the R0-logger instance.
4145 */
4146 VMMRZCallRing3Disable(pVCpu);
4147
4148 /*
4149 * We disable interrupts so that we don't miss any interrupts that would flag preemption
4150 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
4151 * preemption disabled for a while. Since this is purly to aid the
4152 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
4153 * disable interrupt on NT.
4154 *
4155 * We need to check for force-flags that could've possible been altered since we last
4156 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
4157 * see @bugref{6398}).
4158 *
4159 * We also check a couple of other force-flags as a last opportunity to get the EMT back
4160 * to ring-3 before executing guest code.
4161 */
4162 pSvmTransient->fEFlags = ASMIntDisableFlags();
4163 if ( VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
4164 || VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4165 {
4166 ASMSetFlags(pSvmTransient->fEFlags);
4167 VMMRZCallRing3Enable(pVCpu);
4168 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4169 return VINF_EM_RAW_TO_R3;
4170 }
4171 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4172 {
4173 ASMSetFlags(pSvmTransient->fEFlags);
4174 VMMRZCallRing3Enable(pVCpu);
4175 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
4176 return VINF_EM_RAW_INTERRUPT;
4177 }
4178
4179 return VINF_SUCCESS;
4180}
4181
4182
4183/**
4184 * Prepares to run guest (or nested-guest) code in AMD-V and we've committed to
4185 * doing so.
4186 *
4187 * This means there is no backing out to ring-3 or anywhere else at this point.
4188 *
4189 * @param pVCpu The cross context virtual CPU structure.
4190 * @param pSvmTransient Pointer to the SVM transient structure.
4191 *
4192 * @remarks Called with preemption disabled.
4193 * @remarks No-long-jump zone!!!
4194 */
4195static void hmR0SvmPreRunGuestCommitted(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4196{
4197 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4198 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4199
4200 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4201 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4202
4203 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4204 PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4205
4206 hmR0SvmInjectPendingEvent(pVCpu, pVmcb);
4207
4208 if (!CPUMIsGuestFPUStateActive(pVCpu))
4209 {
4210 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4211 CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4212 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
4213 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
4214 }
4215
4216 /* Load the state shared between host and guest (FPU, debug). */
4217 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_SVM_HOST_GUEST_SHARED_STATE)
4218 hmR0SvmExportSharedState(pVCpu, pVmcb);
4219
4220 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT; /* Preemption might set this, nothing to do on AMD-V. */
4221 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
4222
4223 PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
4224 RTCPUID const idHostCpu = pHostCpu->idCpu;
4225 bool const fMigratedHostCpu = idHostCpu != pVCpu->hmr0.s.idLastCpu;
4226
4227 /* Setup TSC offsetting. */
4228 if ( pSvmTransient->fUpdateTscOffsetting
4229 || fMigratedHostCpu)
4230 {
4231 hmR0SvmUpdateTscOffsetting(pVCpu, pVmcb);
4232 pSvmTransient->fUpdateTscOffsetting = false;
4233 }
4234
4235 /* Record statistics of how often we use TSC offsetting as opposed to intercepting RDTSC/P. */
4236 if (!(pVmcb->ctrl.u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC | SVM_CTRL_INTERCEPT_RDTSCP)))
4237 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4238 else
4239 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4240
4241 /* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4242 if (fMigratedHostCpu)
4243 pVmcb->ctrl.u32VmcbCleanBits = 0;
4244
4245 /* Store status of the shared guest-host state at the time of VMRUN. */
4246 pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4247 pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4248
4249#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4250 uint8_t *pbMsrBitmap;
4251 if (!pSvmTransient->fIsNestedGuest)
4252 pbMsrBitmap = (uint8_t *)pVCpu->hmr0.s.svm.pvMsrBitmap;
4253 else
4254 {
4255 /** @todo We could perhaps optimize this by monitoring if the guest modifies its
4256 * MSRPM and only perform this if it changed also use EVEX.POR when it
4257 * does. */
4258 hmR0SvmMergeMsrpmNested(pHostCpu, pVCpu);
4259
4260 /* Update the nested-guest VMCB with the newly merged MSRPM (clean bits updated below). */
4261 pVmcb->ctrl.u64MSRPMPhysAddr = pHostCpu->n.svm.HCPhysNstGstMsrpm;
4262 pbMsrBitmap = (uint8_t *)pHostCpu->n.svm.pvNstGstMsrpm;
4263 }
4264#else
4265 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
4266#endif
4267
4268 ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4269 /* Flush the appropriate tagged-TLB entries. */
4270 hmR0SvmFlushTaggedTlb(pHostCpu, pVCpu, pVmcb);
4271 Assert(pVCpu->hmr0.s.idLastCpu == idHostCpu);
4272
4273 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4274
4275 TMNotifyStartOfExecution(pVM, pVCpu); /* Finally, notify TM to resume its clocks as we're about
4276 to start executing. */
4277
4278 /*
4279 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that RDTSCPs
4280 * (that don't cause exits) reads the guest MSR, see @bugref{3324}.
4281 *
4282 * This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4283 */
4284 if ( pVM->cpum.ro.HostFeatures.fRdTscP
4285 && !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4286 {
4287 uint64_t const uGuestTscAux = CPUMGetGuestTscAux(pVCpu);
4288 pVCpu->hmr0.s.svm.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4289 if (uGuestTscAux != pVCpu->hmr0.s.svm.u64HostTscAux)
4290 ASMWrMsr(MSR_K8_TSC_AUX, uGuestTscAux);
4291 hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4292 pSvmTransient->fRestoreTscAuxMsr = true;
4293 }
4294 else
4295 {
4296 hmR0SvmSetMsrPermission(pVCpu, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4297 pSvmTransient->fRestoreTscAuxMsr = false;
4298 }
4299 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
4300
4301 /*
4302 * If VMCB Clean bits isn't supported by the CPU or exposed to the guest in the nested
4303 * virtualization case, mark all state-bits as dirty indicating to the CPU to re-load
4304 * from the VMCB.
4305 */
4306 bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pSvmTransient->fIsNestedGuest);
4307 if (!fSupportsVmcbCleanBits)
4308 pVmcb->ctrl.u32VmcbCleanBits = 0;
4309}
4310
4311
4312/**
4313 * Wrapper for running the guest (or nested-guest) code in AMD-V.
4314 *
4315 * @returns VBox strict status code.
4316 * @param pVCpu The cross context virtual CPU structure.
4317 * @param HCPhysVmcb The host physical address of the VMCB.
4318 *
4319 * @remarks No-long-jump zone!!!
4320 */
4321DECLINLINE(int) hmR0SvmRunGuest(PVMCPUCC pVCpu, RTHCPHYS HCPhysVmcb)
4322{
4323 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4324 pVCpu->cpum.GstCtx.fExtrn |= HMSVM_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4325 return pVCpu->hmr0.s.svm.pfnVMRun(pVCpu->CTX_SUFF(pVM), pVCpu, HCPhysVmcb);
4326}
4327
4328
4329/**
4330 * Performs some essential restoration of state after running guest (or
4331 * nested-guest) code in AMD-V.
4332 *
4333 * @param pVCpu The cross context virtual CPU structure.
4334 * @param pSvmTransient Pointer to the SVM transient structure.
4335 * @param rcVMRun Return code of VMRUN.
4336 *
4337 * @remarks Called with interrupts disabled.
4338 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
4339 * unconditionally when it is safe to do so.
4340 */
4341static void hmR0SvmPostRunGuest(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, VBOXSTRICTRC rcVMRun)
4342{
4343 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4344
4345 ASMAtomicUoWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4346 ASMAtomicIncU32(&pVCpu->hmr0.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4347
4348 PSVMVMCB pVmcb = pSvmTransient->pVmcb;
4349 PSVMVMCBCTRL pVmcbCtrl = &pVmcb->ctrl;
4350
4351 /* TSC read must be done early for maximum accuracy. */
4352 if (!(pVmcbCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4353 {
4354 if (!pSvmTransient->fIsNestedGuest)
4355 TMCpuTickSetLastSeen(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4356#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4357 else
4358 {
4359 /* The nested-guest VMCB TSC offset shall eventually be restored on #VMEXIT via HMNotifySvmNstGstVmexit(). */
4360 uint64_t const uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, pVCpu->hmr0.s.uTscExit + pVmcbCtrl->u64TSCOffset);
4361 TMCpuTickSetLastSeen(pVCpu, uGstTsc);
4362 }
4363#endif
4364 }
4365
4366 if (pSvmTransient->fRestoreTscAuxMsr)
4367 {
4368 uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4369 CPUMSetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4370 if (u64GuestTscAuxMsr != pVCpu->hmr0.s.svm.u64HostTscAux)
4371 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hmr0.s.svm.u64HostTscAux);
4372 }
4373
4374 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
4375 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4376 TMNotifyEndOfExecution(pVM, pVCpu, pVCpu->hmr0.s.uTscExit); /* Notify TM that the guest is no longer running. */
4377 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4378
4379 Assert(!(ASMGetFlags() & X86_EFL_IF));
4380 ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4381 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4382
4383 /* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4384 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4385 {
4386 Log4Func(("VMRUN failure: rcVMRun=%Rrc\n", VBOXSTRICTRC_VAL(rcVMRun)));
4387 return;
4388 }
4389
4390 pSvmTransient->u64ExitCode = pVmcbCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4391 pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4392 pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4393 pVmcbCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4394
4395#ifdef HMSVM_SYNC_FULL_GUEST_STATE
4396 hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4397 Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMSVM_CPUMCTX_EXTRN_ALL));
4398#else
4399 /*
4400 * Always import the following:
4401 *
4402 * - RIP for exit optimizations and evaluating event injection on re-entry.
4403 * - RFLAGS for evaluating event injection on VM re-entry and for exporting shared debug
4404 * state on preemption.
4405 * - Interrupt shadow, GIF for evaluating event injection on VM re-entry.
4406 * - CS for exit optimizations.
4407 * - RAX, RSP for simplifying assumptions on GPRs. All other GPRs are swapped by the
4408 * assembly switcher code.
4409 * - Shared state (only DR7 currently) for exporting shared debug state on preemption.
4410 */
4411 hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
4412 | CPUMCTX_EXTRN_RFLAGS
4413 | CPUMCTX_EXTRN_RAX
4414 | CPUMCTX_EXTRN_RSP
4415 | CPUMCTX_EXTRN_CS
4416 | CPUMCTX_EXTRN_HWVIRT
4417 | CPUMCTX_EXTRN_INHIBIT_INT
4418 | CPUMCTX_EXTRN_HM_SVM_HWVIRT_VIRQ
4419 | HMSVM_CPUMCTX_SHARED_STATE);
4420#endif
4421
4422 if ( pSvmTransient->u64ExitCode != SVM_EXIT_INVALID
4423 && pVCpu->hmr0.s.svm.fSyncVTpr)
4424 {
4425 Assert(!pSvmTransient->fIsNestedGuest);
4426 /* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4427 if ( pVM->hm.s.fTprPatchingActive
4428 && (pVmcb->guest.u64LSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4429 {
4430 int rc = APICSetTpr(pVCpu, pVmcb->guest.u64LSTAR & 0xff);
4431 AssertRC(rc);
4432 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4433 }
4434 /* Sync TPR when we aren't intercepting CR8 writes. */
4435 else if (pSvmTransient->u8GuestTpr != pVmcbCtrl->IntCtrl.n.u8VTPR)
4436 {
4437 int rc = APICSetTpr(pVCpu, pVmcbCtrl->IntCtrl.n.u8VTPR << 4);
4438 AssertRC(rc);
4439 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
4440 }
4441 }
4442
4443#ifdef DEBUG_ramshankar
4444 if (CPUMIsGuestInSvmNestedHwVirtMode(&pVCpu->cpum.GstCtx))
4445 {
4446 hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4447 hmR0SvmLogState(pVCpu, pVmcb, pVCpu->cpum.GstCtx, "hmR0SvmPostRunGuestNested", HMSVM_LOG_ALL & ~HMSVM_LOG_LBR,
4448 0 /* uVerbose */);
4449 }
4450#endif
4451
4452 HMSVM_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
4453 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_SVM, pSvmTransient->u64ExitCode & EMEXIT_F_TYPE_MASK),
4454 pVCpu->cpum.GstCtx.cs.u64Base + pVCpu->cpum.GstCtx.rip, pVCpu->hmr0.s.uTscExit);
4455}
4456
4457
4458/**
4459 * Runs the guest code using AMD-V.
4460 *
4461 * @returns Strict VBox status code.
4462 * @param pVCpu The cross context virtual CPU structure.
4463 * @param pcLoops Pointer to the number of executed loops.
4464 */
4465static VBOXSTRICTRC hmR0SvmRunGuestCodeNormal(PVMCPUCC pVCpu, uint32_t *pcLoops)
4466{
4467 uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
4468 Assert(pcLoops);
4469 Assert(*pcLoops <= cMaxResumeLoops);
4470
4471 SVMTRANSIENT SvmTransient;
4472 RT_ZERO(SvmTransient);
4473 SvmTransient.fUpdateTscOffsetting = true;
4474 SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
4475
4476 VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
4477 for (;;)
4478 {
4479 Assert(!HMR0SuspendPending());
4480 HMSVM_ASSERT_CPU_SAFE(pVCpu);
4481
4482 /* Preparatory work for running nested-guest code, this may force us to return to
4483 ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4484 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4485 rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4486 if (rc != VINF_SUCCESS)
4487 break;
4488
4489 /*
4490 * No longjmps to ring-3 from this point on!!!
4491 *
4492 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4493 * better than a kernel panic. This also disables flushing of the R0-logger instance.
4494 */
4495 hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4496 rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
4497
4498 /* Restore any residual host-state and save any bits shared between host and guest
4499 into the guest-CPU state. Re-enables interrupts! */
4500 hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4501
4502 if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4503 || SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4504 {
4505 if (rc == VINF_SUCCESS)
4506 rc = VERR_SVM_INVALID_GUEST_STATE;
4507 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
4508 hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
4509 break;
4510 }
4511
4512 /* Handle the #VMEXIT. */
4513 HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4514 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4515 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
4516 rc = hmR0SvmHandleExit(pVCpu, &SvmTransient);
4517 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4518 if (rc != VINF_SUCCESS)
4519 break;
4520 if (++(*pcLoops) >= cMaxResumeLoops)
4521 {
4522 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4523 rc = VINF_EM_RAW_INTERRUPT;
4524 break;
4525 }
4526 }
4527
4528 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4529 return rc;
4530}
4531
4532
4533#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4534/**
4535 * Runs the nested-guest code using AMD-V.
4536 *
4537 * @returns Strict VBox status code.
4538 * @param pVCpu The cross context virtual CPU structure.
4539 * @param pcLoops Pointer to the number of executed loops. If we're switching
4540 * from the guest-code execution loop to this nested-guest
4541 * execution loop pass the remainder value, else pass 0.
4542 */
4543static VBOXSTRICTRC hmR0SvmRunGuestCodeNested(PVMCPUCC pVCpu, uint32_t *pcLoops)
4544{
4545 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4546 HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4547 Assert(pcLoops);
4548 Assert(*pcLoops <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops);
4549 /** @todo r=bird: Sharing this with ring-3 isn't safe in the long run, I fear... */
4550 RTHCPHYS const HCPhysVmcb = GVMMR0ConvertGVMPtr2HCPhys(pVCpu->pGVM, &pCtx->hwvirt.svm.Vmcb);
4551
4552 SVMTRANSIENT SvmTransient;
4553 RT_ZERO(SvmTransient);
4554 SvmTransient.fUpdateTscOffsetting = true;
4555 SvmTransient.pVmcb = &pCtx->hwvirt.svm.Vmcb;
4556 SvmTransient.fIsNestedGuest = true;
4557
4558 VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_4;
4559 for (;;)
4560 {
4561 Assert(!HMR0SuspendPending());
4562 HMSVM_ASSERT_CPU_SAFE(pVCpu);
4563
4564 /* Preparatory work for running nested-guest code, this may force us to return to
4565 ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4566 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4567 rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
4568 if ( rc != VINF_SUCCESS
4569 || !CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4570 break;
4571
4572 /*
4573 * No longjmps to ring-3 from this point on!!!
4574 *
4575 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
4576 * better than a kernel panic. This also disables flushing of the R0-logger instance.
4577 */
4578 hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
4579
4580 rc = hmR0SvmRunGuest(pVCpu, HCPhysVmcb);
4581
4582 /* Restore any residual host-state and save any bits shared between host and guest
4583 into the guest-CPU state. Re-enables interrupts! */
4584 hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
4585
4586 if (RT_LIKELY( rc == VINF_SUCCESS
4587 && SvmTransient.u64ExitCode != SVM_EXIT_INVALID))
4588 { /* extremely likely */ }
4589 else
4590 {
4591 /* VMRUN failed, shouldn't really happen, Guru. */
4592 if (rc != VINF_SUCCESS)
4593 break;
4594
4595 /* Invalid nested-guest state. Cause a #VMEXIT but assert on strict builds. */
4596 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
4597 AssertMsgFailed(("Invalid nested-guest state. rc=%Rrc u64ExitCode=%#RX64\n", rc, SvmTransient.u64ExitCode));
4598 rc = IEMExecSvmVmexit(pVCpu, SVM_EXIT_INVALID, 0, 0);
4599 break;
4600 }
4601
4602 /* Handle the #VMEXIT. */
4603 HMSVM_NESTED_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4604 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
4605 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, &pCtx->hwvirt.svm.Vmcb);
4606 rc = hmR0SvmHandleExitNested(pVCpu, &SvmTransient);
4607 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
4608 if (rc == VINF_SUCCESS)
4609 {
4610 if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4611 {
4612 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchNstGstVmexit);
4613 rc = VINF_SVM_VMEXIT;
4614 }
4615 else
4616 {
4617 if (++(*pcLoops) <= pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops)
4618 continue;
4619 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4620 rc = VINF_EM_RAW_INTERRUPT;
4621 }
4622 }
4623 else
4624 Assert(rc != VINF_SVM_VMEXIT);
4625 break;
4626 /** @todo NSTSVM: handle single-stepping. */
4627 }
4628
4629 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4630 return rc;
4631}
4632#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
4633
4634
4635/**
4636 * Checks if any expensive dtrace probes are enabled and we should go to the
4637 * debug loop.
4638 *
4639 * @returns true if we should use debug loop, false if not.
4640 */
4641static bool hmR0SvmAnyExpensiveProbesEnabled(void)
4642{
4643 /* It's probably faster to OR the raw 32-bit counter variables together.
4644 Since the variables are in an array and the probes are next to one
4645 another (more or less), we have good locality. So, better read
4646 eight-nine cache lines ever time and only have one conditional, than
4647 128+ conditionals, right? */
4648 return ( VBOXVMM_R0_HMSVM_VMEXIT_ENABLED_RAW() /* expensive too due to context */
4649 | VBOXVMM_XCPT_DE_ENABLED_RAW()
4650 | VBOXVMM_XCPT_DB_ENABLED_RAW()
4651 | VBOXVMM_XCPT_BP_ENABLED_RAW()
4652 | VBOXVMM_XCPT_OF_ENABLED_RAW()
4653 | VBOXVMM_XCPT_BR_ENABLED_RAW()
4654 | VBOXVMM_XCPT_UD_ENABLED_RAW()
4655 | VBOXVMM_XCPT_NM_ENABLED_RAW()
4656 | VBOXVMM_XCPT_DF_ENABLED_RAW()
4657 | VBOXVMM_XCPT_TS_ENABLED_RAW()
4658 | VBOXVMM_XCPT_NP_ENABLED_RAW()
4659 | VBOXVMM_XCPT_SS_ENABLED_RAW()
4660 | VBOXVMM_XCPT_GP_ENABLED_RAW()
4661 | VBOXVMM_XCPT_PF_ENABLED_RAW()
4662 | VBOXVMM_XCPT_MF_ENABLED_RAW()
4663 | VBOXVMM_XCPT_AC_ENABLED_RAW()
4664 | VBOXVMM_XCPT_XF_ENABLED_RAW()
4665 | VBOXVMM_XCPT_VE_ENABLED_RAW()
4666 | VBOXVMM_XCPT_SX_ENABLED_RAW()
4667 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
4668 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
4669 ) != 0
4670 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
4671 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
4672 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
4673 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
4674 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
4675 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
4676 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
4677 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
4678 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
4679 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
4680 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
4681 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
4682 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
4683 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
4684 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
4685 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
4686 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
4687 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
4688 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
4689 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
4690 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
4691 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
4692 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
4693 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
4694 | VBOXVMM_INSTR_STR_ENABLED_RAW()
4695 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
4696 //| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
4697 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
4698 //| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
4699 //| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
4700 //| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
4701 //| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
4702 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
4703 | VBOXVMM_INSTR_SVM_VMRUN_ENABLED_RAW()
4704 | VBOXVMM_INSTR_SVM_VMLOAD_ENABLED_RAW()
4705 | VBOXVMM_INSTR_SVM_VMSAVE_ENABLED_RAW()
4706 | VBOXVMM_INSTR_SVM_STGI_ENABLED_RAW()
4707 | VBOXVMM_INSTR_SVM_CLGI_ENABLED_RAW()
4708 ) != 0
4709 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
4710 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
4711 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
4712 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
4713 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
4714 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
4715 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
4716 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
4717 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
4718 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
4719 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
4720 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
4721 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
4722 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
4723 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
4724 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
4725 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
4726 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
4727 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
4728 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
4729 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
4730 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
4731 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
4732 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
4733 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
4734 | VBOXVMM_EXIT_STR_ENABLED_RAW()
4735 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
4736 //| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
4737 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
4738 //| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
4739 //| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
4740 //| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
4741 //| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
4742 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
4743 | VBOXVMM_EXIT_SVM_VMRUN_ENABLED_RAW()
4744 | VBOXVMM_EXIT_SVM_VMLOAD_ENABLED_RAW()
4745 | VBOXVMM_EXIT_SVM_VMSAVE_ENABLED_RAW()
4746 | VBOXVMM_EXIT_SVM_STGI_ENABLED_RAW()
4747 | VBOXVMM_EXIT_SVM_CLGI_ENABLED_RAW()
4748 ) != 0;
4749}
4750
4751
4752/**
4753 * Runs the guest code using AMD-V.
4754 *
4755 * @returns Strict VBox status code.
4756 * @param pVCpu The cross context virtual CPU structure.
4757 */
4758VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVMCPUCC pVCpu)
4759{
4760 AssertPtr(pVCpu);
4761 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4762 Assert(VMMRZCallRing3IsEnabled(pVCpu));
4763 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4764 HMSVM_ASSERT_PREEMPT_SAFE(pVCpu);
4765
4766 uint32_t cLoops = 0;
4767 VBOXSTRICTRC rc;
4768 for (;;)
4769 {
4770#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4771 bool const fInNestedGuestMode = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
4772#else
4773 NOREF(pCtx);
4774 bool const fInNestedGuestMode = false;
4775#endif
4776 if (!fInNestedGuestMode)
4777 {
4778 if ( !pVCpu->hm.s.fUseDebugLoop
4779 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0SvmAnyExpensiveProbesEnabled())
4780 && !DBGFIsStepping(pVCpu)
4781 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
4782 rc = hmR0SvmRunGuestCodeNormal(pVCpu, &cLoops);
4783 else
4784 rc = hmR0SvmRunGuestCodeDebug(pVCpu, &cLoops);
4785 }
4786#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4787 else
4788 rc = hmR0SvmRunGuestCodeNested(pVCpu, &cLoops);
4789
4790 if (rc == VINF_SVM_VMRUN)
4791 {
4792 Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4793 continue;
4794 }
4795 if (rc == VINF_SVM_VMEXIT)
4796 {
4797 Assert(!CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
4798 continue;
4799 }
4800#endif
4801 break;
4802 }
4803
4804 /* Fixup error codes. */
4805 if (rc == VERR_EM_INTERPRETER)
4806 rc = VINF_EM_RAW_EMULATE_INSTR;
4807 else if (rc == VINF_EM_RESET)
4808 rc = VINF_EM_TRIPLE_FAULT;
4809
4810 /* Prepare to return to ring-3. This will remove longjmp notifications. */
4811 rc = hmR0SvmExitToRing3(pVCpu, rc);
4812 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
4813 Assert(!VMMR0AssertionIsNotificationSet(pVCpu));
4814 return rc;
4815}
4816
4817#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
4818
4819/**
4820 * Determines whether the given I/O access should cause a nested-guest \#VMEXIT.
4821 *
4822 * @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4823 * @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4824 */
4825static bool hmR0SvmIsIoInterceptSet(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4826{
4827 const uint16_t u16Port = pIoExitInfo->n.u16Port;
4828 const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4829 const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4830 const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4831 const uint8_t iEffSeg = pIoExitInfo->n.u3Seg;
4832 const bool fRep = pIoExitInfo->n.u1Rep;
4833 const bool fStrIo = pIoExitInfo->n.u1Str;
4834
4835 return CPUMIsSvmIoInterceptSet(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4836 NULL /* pIoExitInfo */);
4837}
4838
4839
4840/**
4841 * Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4842 * SVM_EXIT_INVALID).
4843 *
4844 * @returns VBox status code (informational status codes included).
4845 * @param pVCpu The cross context virtual CPU structure.
4846 * @param pSvmTransient Pointer to the SVM transient structure.
4847 */
4848static VBOXSTRICTRC hmR0SvmHandleExitNested(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
4849{
4850 HMSVM_ASSERT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
4851 Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4852 Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4853
4854 /*
4855 * We import the complete state here because we use separate VMCBs for the guest and the
4856 * nested-guest, and the guest's VMCB is used after the #VMEXIT. We can only save/restore
4857 * the #VMEXIT specific state if we used the same VMCB for both guest and nested-guest.
4858 */
4859#define NST_GST_VMEXIT_CALL_RET(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4860 do { \
4861 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
4862 return IEMExecSvmVmexit((a_pVCpu), (a_uExitCode), (a_uExitInfo1), (a_uExitInfo2)); \
4863 } while (0)
4864
4865 /*
4866 * For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected by the
4867 * nested-guest. If it isn't, it should be handled by the (outer) guest.
4868 */
4869 PSVMVMCB pVmcbNstGst = &pVCpu->cpum.GstCtx.hwvirt.svm.Vmcb;
4870 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4871 PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4872 uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4873 uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4874 uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4875
4876 Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4877 switch (uExitCode)
4878 {
4879 case SVM_EXIT_CPUID:
4880 {
4881 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4882 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4883 return hmR0SvmExitCpuid(pVCpu, pSvmTransient);
4884 }
4885
4886 case SVM_EXIT_RDTSC:
4887 {
4888 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4889 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4890 return hmR0SvmExitRdtsc(pVCpu, pSvmTransient);
4891 }
4892
4893 case SVM_EXIT_RDTSCP:
4894 {
4895 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4896 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4897 return hmR0SvmExitRdtscp(pVCpu, pSvmTransient);
4898 }
4899
4900 case SVM_EXIT_MONITOR:
4901 {
4902 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4903 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4904 return hmR0SvmExitMonitor(pVCpu, pSvmTransient);
4905 }
4906
4907 case SVM_EXIT_MWAIT:
4908 {
4909 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4910 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4911 return hmR0SvmExitMwait(pVCpu, pSvmTransient);
4912 }
4913
4914 case SVM_EXIT_HLT:
4915 {
4916 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4917 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4918 return hmR0SvmExitHlt(pVCpu, pSvmTransient);
4919 }
4920
4921 case SVM_EXIT_MSR:
4922 {
4923 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4924 {
4925 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
4926 uint16_t offMsrpm;
4927 uint8_t uMsrpmBit;
4928 int rc = CPUMGetSvmMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4929 if (RT_SUCCESS(rc))
4930 {
4931 Assert(uMsrpmBit == 0 || uMsrpmBit == 2 || uMsrpmBit == 4 || uMsrpmBit == 6);
4932 Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
4933
4934 uint8_t const * const pbMsrBitmap = &pVCpu->cpum.GstCtx.hwvirt.svm.abMsrBitmap[offMsrpm];
4935 bool const fInterceptRead = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit));
4936 bool const fInterceptWrite = RT_BOOL(*pbMsrBitmap & RT_BIT(uMsrpmBit + 1));
4937
4938 if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4939 || (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4940 {
4941 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4942 }
4943 }
4944 else
4945 {
4946 /*
4947 * MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4948 * See AMD-V spec. "15.11 MSR Intercepts".
4949 */
4950 Assert(rc == VERR_OUT_OF_RANGE);
4951 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4952 }
4953 }
4954 return hmR0SvmExitMsr(pVCpu, pSvmTransient);
4955 }
4956
4957 case SVM_EXIT_IOIO:
4958 {
4959 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4960 {
4961 SVMIOIOEXITINFO IoExitInfo;
4962 IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4963 bool const fIntercept = hmR0SvmIsIoInterceptSet(pVCpu->cpum.GstCtx.hwvirt.svm.abIoBitmap, &IoExitInfo);
4964 if (fIntercept)
4965 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4966 }
4967 return hmR0SvmExitIOInstr(pVCpu, pSvmTransient);
4968 }
4969
4970 case SVM_EXIT_XCPT_PF:
4971 {
4972 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4973 if (pVM->hmr0.s.fNestedPaging)
4974 {
4975 uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4976 uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4977
4978 /* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4979 if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4980 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4981
4982 /* If the nested-guest is not intercepting #PFs, forward the #PF to the guest. */
4983 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
4984 hmR0SvmSetPendingXcptPF(pVCpu, u32ErrCode, uFaultAddress);
4985 return VINF_SUCCESS;
4986 }
4987 return hmR0SvmExitXcptPF(pVCpu, pSvmTransient);
4988 }
4989
4990 case SVM_EXIT_XCPT_UD:
4991 {
4992 if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4993 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4994 hmR0SvmSetPendingXcptUD(pVCpu);
4995 return VINF_SUCCESS;
4996 }
4997
4998 case SVM_EXIT_XCPT_MF:
4999 {
5000 if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
5001 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5002 return hmR0SvmExitXcptMF(pVCpu, pSvmTransient);
5003 }
5004
5005 case SVM_EXIT_XCPT_DB:
5006 {
5007 if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
5008 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5009 return hmR0SvmNestedExitXcptDB(pVCpu, pSvmTransient);
5010 }
5011
5012 case SVM_EXIT_XCPT_AC:
5013 {
5014 if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
5015 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5016 return hmR0SvmExitXcptAC(pVCpu, pSvmTransient);
5017 }
5018
5019 case SVM_EXIT_XCPT_BP:
5020 {
5021 if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5022 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5023 return hmR0SvmNestedExitXcptBP(pVCpu, pSvmTransient);
5024 }
5025
5026 case SVM_EXIT_READ_CR0:
5027 case SVM_EXIT_READ_CR3:
5028 case SVM_EXIT_READ_CR4:
5029 {
5030 uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5031 if (CPUMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5032 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5033 return hmR0SvmExitReadCRx(pVCpu, pSvmTransient);
5034 }
5035
5036 case SVM_EXIT_CR0_SEL_WRITE:
5037 {
5038 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CR0_SEL_WRITE))
5039 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5040 return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5041 }
5042
5043 case SVM_EXIT_WRITE_CR0:
5044 case SVM_EXIT_WRITE_CR3:
5045 case SVM_EXIT_WRITE_CR4:
5046 case SVM_EXIT_WRITE_CR8: /* CR8 writes would go to the V_TPR rather than here, since we run with V_INTR_MASKING. */
5047 {
5048 uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5049 Log4Func(("Write CR%u: uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", uCr, uExitInfo1, uExitInfo2));
5050
5051 if (CPUMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5052 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5053 return hmR0SvmExitWriteCRx(pVCpu, pSvmTransient);
5054 }
5055
5056 case SVM_EXIT_PAUSE:
5057 {
5058 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5059 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5060 return hmR0SvmExitPause(pVCpu, pSvmTransient);
5061 }
5062
5063 case SVM_EXIT_VINTR:
5064 {
5065 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5066 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5067 return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5068 }
5069
5070 case SVM_EXIT_INTR:
5071 case SVM_EXIT_NMI:
5072 case SVM_EXIT_SMI:
5073 case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5074 {
5075 /*
5076 * We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5077 *
5078 * Although we don't intercept SMIs, the nested-guest might. Therefore, we might
5079 * get an SMI #VMEXIT here so simply ignore rather than causing a corresponding
5080 * nested-guest #VMEXIT.
5081 *
5082 * We shall import the complete state here as we may cause #VMEXITs from ring-3
5083 * while trying to inject interrupts, see comment at the top of this function.
5084 */
5085 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_ALL);
5086 return hmR0SvmExitIntr(pVCpu, pSvmTransient);
5087 }
5088
5089 case SVM_EXIT_FERR_FREEZE:
5090 {
5091 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5092 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5093 return hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient);
5094 }
5095
5096 case SVM_EXIT_INVLPG:
5097 {
5098 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5099 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5100 return hmR0SvmExitInvlpg(pVCpu, pSvmTransient);
5101 }
5102
5103 case SVM_EXIT_WBINVD:
5104 {
5105 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5106 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5107 return hmR0SvmExitWbinvd(pVCpu, pSvmTransient);
5108 }
5109
5110 case SVM_EXIT_INVD:
5111 {
5112 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5113 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5114 return hmR0SvmExitInvd(pVCpu, pSvmTransient);
5115 }
5116
5117 case SVM_EXIT_RDPMC:
5118 {
5119 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5120 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5121 return hmR0SvmExitRdpmc(pVCpu, pSvmTransient);
5122 }
5123
5124 default:
5125 {
5126 switch (uExitCode)
5127 {
5128 case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5129 case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5130 case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5131 case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5132 {
5133 uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5134 if (CPUMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5135 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5136 return hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
5137 }
5138
5139 case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5140 case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5141 case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5142 case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5143 {
5144 uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5145 if (CPUMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5146 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5147 return hmR0SvmExitWriteDRx(pVCpu, pSvmTransient);
5148 }
5149
5150 case SVM_EXIT_XCPT_DE:
5151 /* SVM_EXIT_XCPT_DB: */ /* Handled above. */
5152 /* SVM_EXIT_XCPT_NMI: */ /* Handled above. */
5153 /* SVM_EXIT_XCPT_BP: */ /* Handled above. */
5154 case SVM_EXIT_XCPT_OF:
5155 case SVM_EXIT_XCPT_BR:
5156 /* SVM_EXIT_XCPT_UD: */ /* Handled above. */
5157 case SVM_EXIT_XCPT_NM:
5158 case SVM_EXIT_XCPT_DF:
5159 case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5160 case SVM_EXIT_XCPT_TS:
5161 case SVM_EXIT_XCPT_NP:
5162 case SVM_EXIT_XCPT_SS:
5163 case SVM_EXIT_XCPT_GP:
5164 /* SVM_EXIT_XCPT_PF: */ /* Handled above. */
5165 case SVM_EXIT_XCPT_15: /* Reserved. */
5166 /* SVM_EXIT_XCPT_MF: */ /* Handled above. */
5167 /* SVM_EXIT_XCPT_AC: */ /* Handled above. */
5168 case SVM_EXIT_XCPT_MC:
5169 case SVM_EXIT_XCPT_XF:
5170 case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5171 case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5172 case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5173 {
5174 uint8_t const uVector = uExitCode - SVM_EXIT_XCPT_0;
5175 if (CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5176 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5177 return hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient);
5178 }
5179
5180 case SVM_EXIT_XSETBV:
5181 {
5182 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5183 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5184 return hmR0SvmExitXsetbv(pVCpu, pSvmTransient);
5185 }
5186
5187 case SVM_EXIT_TASK_SWITCH:
5188 {
5189 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5190 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5191 return hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient);
5192 }
5193
5194 case SVM_EXIT_IRET:
5195 {
5196 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5197 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5198 return hmR0SvmExitIret(pVCpu, pSvmTransient);
5199 }
5200
5201 case SVM_EXIT_SHUTDOWN:
5202 {
5203 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5204 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5205 return hmR0SvmExitShutdown(pVCpu, pSvmTransient);
5206 }
5207
5208 case SVM_EXIT_VMMCALL:
5209 {
5210 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5211 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5212 return hmR0SvmExitVmmCall(pVCpu, pSvmTransient);
5213 }
5214
5215 case SVM_EXIT_CLGI:
5216 {
5217 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5218 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5219 return hmR0SvmExitClgi(pVCpu, pSvmTransient);
5220 }
5221
5222 case SVM_EXIT_STGI:
5223 {
5224 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5225 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5226 return hmR0SvmExitStgi(pVCpu, pSvmTransient);
5227 }
5228
5229 case SVM_EXIT_VMLOAD:
5230 {
5231 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5232 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5233 return hmR0SvmExitVmload(pVCpu, pSvmTransient);
5234 }
5235
5236 case SVM_EXIT_VMSAVE:
5237 {
5238 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5239 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5240 return hmR0SvmExitVmsave(pVCpu, pSvmTransient);
5241 }
5242
5243 case SVM_EXIT_INVLPGA:
5244 {
5245 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5246 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5247 return hmR0SvmExitInvlpga(pVCpu, pSvmTransient);
5248 }
5249
5250 case SVM_EXIT_VMRUN:
5251 {
5252 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5253 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5254 return hmR0SvmExitVmrun(pVCpu, pSvmTransient);
5255 }
5256
5257 case SVM_EXIT_RSM:
5258 {
5259 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5260 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5261 hmR0SvmSetPendingXcptUD(pVCpu);
5262 return VINF_SUCCESS;
5263 }
5264
5265 case SVM_EXIT_SKINIT:
5266 {
5267 if (CPUMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5268 NST_GST_VMEXIT_CALL_RET(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5269 hmR0SvmSetPendingXcptUD(pVCpu);
5270 return VINF_SUCCESS;
5271 }
5272
5273 case SVM_EXIT_NPF:
5274 {
5275 Assert(pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
5276 return hmR0SvmExitNestedPF(pVCpu, pSvmTransient);
5277 }
5278
5279 case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5280 return hmR0SvmExitUnexpected(pVCpu, pSvmTransient);
5281
5282 default:
5283 {
5284 AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5285 pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5286 return VERR_SVM_UNKNOWN_EXIT;
5287 }
5288 }
5289 }
5290 }
5291 /* not reached */
5292
5293# undef NST_GST_VMEXIT_CALL_RET
5294}
5295
5296#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
5297
5298/** @def VMEXIT_CALL_RET
5299 * Used by hmR0SvmHandleExit and hmR0SvmDebugHandleExit
5300 */
5301#ifdef DEBUG_ramshankar
5302# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) \
5303 do { \
5304 if ((a_fDbg) == 1) \
5305 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL); \
5306 int rc = a_CallExpr; \
5307 if ((a_fDbg) == 1) \
5308 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
5309 return rc; \
5310 } while (0)
5311#else
5312# define VMEXIT_CALL_RET(a_fDbg, a_CallExpr) return a_CallExpr
5313#endif
5314
5315/**
5316 * Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5317 *
5318 * @returns Strict VBox status code (informational status codes included).
5319 * @param pVCpu The cross context virtual CPU structure.
5320 * @param pSvmTransient Pointer to the SVM transient structure.
5321 */
5322static VBOXSTRICTRC hmR0SvmHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
5323{
5324 Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5325 Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5326
5327 /*
5328 * The ordering of the case labels is based on most-frequently-occurring #VMEXITs
5329 * for most guests under normal workloads (for some definition of "normal").
5330 */
5331 uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5332 switch (uExitCode)
5333 {
5334 case SVM_EXIT_NPF: VMEXIT_CALL_RET(0, hmR0SvmExitNestedPF(pVCpu, pSvmTransient));
5335 case SVM_EXIT_IOIO: VMEXIT_CALL_RET(0, hmR0SvmExitIOInstr(pVCpu, pSvmTransient));
5336 case SVM_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0SvmExitRdtsc(pVCpu, pSvmTransient));
5337 case SVM_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0SvmExitRdtscp(pVCpu, pSvmTransient));
5338 case SVM_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0SvmExitCpuid(pVCpu, pSvmTransient));
5339 case SVM_EXIT_XCPT_PF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptPF(pVCpu, pSvmTransient));
5340 case SVM_EXIT_MSR: VMEXIT_CALL_RET(0, hmR0SvmExitMsr(pVCpu, pSvmTransient));
5341 case SVM_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0SvmExitMonitor(pVCpu, pSvmTransient));
5342 case SVM_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0SvmExitMwait(pVCpu, pSvmTransient));
5343 case SVM_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0SvmExitHlt(pVCpu, pSvmTransient));
5344
5345 case SVM_EXIT_XCPT_NMI: /* Should not occur, SVM_EXIT_NMI is used instead. */
5346 case SVM_EXIT_INTR:
5347 case SVM_EXIT_NMI: VMEXIT_CALL_RET(0, hmR0SvmExitIntr(pVCpu, pSvmTransient));
5348
5349 case SVM_EXIT_READ_CR0:
5350 case SVM_EXIT_READ_CR3:
5351 case SVM_EXIT_READ_CR4: VMEXIT_CALL_RET(0, hmR0SvmExitReadCRx(pVCpu, pSvmTransient));
5352
5353 case SVM_EXIT_CR0_SEL_WRITE:
5354 case SVM_EXIT_WRITE_CR0:
5355 case SVM_EXIT_WRITE_CR3:
5356 case SVM_EXIT_WRITE_CR4:
5357 case SVM_EXIT_WRITE_CR8: VMEXIT_CALL_RET(0, hmR0SvmExitWriteCRx(pVCpu, pSvmTransient));
5358
5359 case SVM_EXIT_VINTR: VMEXIT_CALL_RET(0, hmR0SvmExitVIntr(pVCpu, pSvmTransient));
5360 case SVM_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0SvmExitPause(pVCpu, pSvmTransient));
5361 case SVM_EXIT_VMMCALL: VMEXIT_CALL_RET(0, hmR0SvmExitVmmCall(pVCpu, pSvmTransient));
5362 case SVM_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpg(pVCpu, pSvmTransient));
5363 case SVM_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0SvmExitWbinvd(pVCpu, pSvmTransient));
5364 case SVM_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0SvmExitInvd(pVCpu, pSvmTransient));
5365 case SVM_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0SvmExitRdpmc(pVCpu, pSvmTransient));
5366 case SVM_EXIT_IRET: VMEXIT_CALL_RET(0, hmR0SvmExitIret(pVCpu, pSvmTransient));
5367 case SVM_EXIT_XCPT_UD: VMEXIT_CALL_RET(0, hmR0SvmExitXcptUD(pVCpu, pSvmTransient));
5368 case SVM_EXIT_XCPT_MF: VMEXIT_CALL_RET(0, hmR0SvmExitXcptMF(pVCpu, pSvmTransient));
5369 case SVM_EXIT_XCPT_DB: VMEXIT_CALL_RET(0, hmR0SvmExitXcptDB(pVCpu, pSvmTransient));
5370 case SVM_EXIT_XCPT_AC: VMEXIT_CALL_RET(0, hmR0SvmExitXcptAC(pVCpu, pSvmTransient));
5371 case SVM_EXIT_XCPT_BP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptBP(pVCpu, pSvmTransient));
5372 case SVM_EXIT_XCPT_GP: VMEXIT_CALL_RET(0, hmR0SvmExitXcptGP(pVCpu, pSvmTransient));
5373 case SVM_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0SvmExitXsetbv(pVCpu, pSvmTransient));
5374 case SVM_EXIT_FERR_FREEZE: VMEXIT_CALL_RET(0, hmR0SvmExitFerrFreeze(pVCpu, pSvmTransient));
5375
5376 default:
5377 {
5378 switch (pSvmTransient->u64ExitCode)
5379 {
5380 case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5381 case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5382 case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5383 case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5384 VMEXIT_CALL_RET(0, hmR0SvmExitReadDRx(pVCpu, pSvmTransient));
5385
5386 case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5387 case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5388 case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5389 case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5390 VMEXIT_CALL_RET(0, hmR0SvmExitWriteDRx(pVCpu, pSvmTransient));
5391
5392 case SVM_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0SvmExitTaskSwitch(pVCpu, pSvmTransient));
5393 case SVM_EXIT_SHUTDOWN: VMEXIT_CALL_RET(0, hmR0SvmExitShutdown(pVCpu, pSvmTransient));
5394
5395 case SVM_EXIT_SMI:
5396 case SVM_EXIT_INIT:
5397 {
5398 /*
5399 * We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5400 * If it ever does, we want to know about it so log the exit code and bail.
5401 */
5402 VMEXIT_CALL_RET(0, hmR0SvmExitUnexpected(pVCpu, pSvmTransient));
5403 }
5404
5405#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
5406 case SVM_EXIT_CLGI: VMEXIT_CALL_RET(0, hmR0SvmExitClgi(pVCpu, pSvmTransient));
5407 case SVM_EXIT_STGI: VMEXIT_CALL_RET(0, hmR0SvmExitStgi(pVCpu, pSvmTransient));
5408 case SVM_EXIT_VMLOAD: VMEXIT_CALL_RET(0, hmR0SvmExitVmload(pVCpu, pSvmTransient));
5409 case SVM_EXIT_VMSAVE: VMEXIT_CALL_RET(0, hmR0SvmExitVmsave(pVCpu, pSvmTransient));
5410 case SVM_EXIT_INVLPGA: VMEXIT_CALL_RET(0, hmR0SvmExitInvlpga(pVCpu, pSvmTransient));
5411 case SVM_EXIT_VMRUN: VMEXIT_CALL_RET(0, hmR0SvmExitVmrun(pVCpu, pSvmTransient));
5412#else
5413 case SVM_EXIT_CLGI:
5414 case SVM_EXIT_STGI:
5415 case SVM_EXIT_VMLOAD:
5416 case SVM_EXIT_VMSAVE:
5417 case SVM_EXIT_INVLPGA:
5418 case SVM_EXIT_VMRUN:
5419#endif
5420 case SVM_EXIT_RSM:
5421 case SVM_EXIT_SKINIT:
5422 {
5423 hmR0SvmSetPendingXcptUD(pVCpu);
5424 return VINF_SUCCESS;
5425 }
5426
5427 /*
5428 * The remaining should only be possible when debugging or dtracing.
5429 */
5430 case SVM_EXIT_XCPT_DE:
5431 /* SVM_EXIT_XCPT_DB: */ /* Handled above. */
5432 /* SVM_EXIT_XCPT_NMI: */ /* Handled above. */
5433 /* SVM_EXIT_XCPT_BP: */ /* Handled above. */
5434 case SVM_EXIT_XCPT_OF:
5435 case SVM_EXIT_XCPT_BR:
5436 /* SVM_EXIT_XCPT_UD: */ /* Handled above. */
5437 case SVM_EXIT_XCPT_NM:
5438 case SVM_EXIT_XCPT_DF:
5439 case SVM_EXIT_XCPT_CO_SEG_OVERRUN:
5440 case SVM_EXIT_XCPT_TS:
5441 case SVM_EXIT_XCPT_NP:
5442 case SVM_EXIT_XCPT_SS:
5443 /* SVM_EXIT_XCPT_GP: */ /* Handled above. */
5444 /* SVM_EXIT_XCPT_PF: */
5445 case SVM_EXIT_XCPT_15: /* Reserved. */
5446 /* SVM_EXIT_XCPT_MF: */ /* Handled above. */
5447 /* SVM_EXIT_XCPT_AC: */ /* Handled above. */
5448 case SVM_EXIT_XCPT_MC:
5449 case SVM_EXIT_XCPT_XF:
5450 case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
5451 case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
5452 case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
5453 VMEXIT_CALL_RET(0, hmR0SvmExitXcptGeneric(pVCpu, pSvmTransient));
5454
5455 case SVM_EXIT_SWINT: VMEXIT_CALL_RET(0, hmR0SvmExitSwInt(pVCpu, pSvmTransient));
5456 case SVM_EXIT_TR_READ: VMEXIT_CALL_RET(0, hmR0SvmExitTrRead(pVCpu, pSvmTransient));
5457 case SVM_EXIT_TR_WRITE: VMEXIT_CALL_RET(0, hmR0SvmExitTrWrite(pVCpu, pSvmTransient)); /* Also OS/2 TLB workaround. */
5458
5459 default:
5460 {
5461 AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5462 pVCpu->hm.s.u32HMError = uExitCode;
5463 return VERR_SVM_UNKNOWN_EXIT;
5464 }
5465 }
5466 }
5467 }
5468 /* not reached */
5469}
5470
5471
5472/** @name Execution loop for single stepping, DBGF events and expensive Dtrace probes.
5473 *
5474 * The following few functions and associated structure contains the bloat
5475 * necessary for providing detailed debug events and dtrace probes as well as
5476 * reliable host side single stepping. This works on the principle of
5477 * "subclassing" the normal execution loop and workers. We replace the loop
5478 * method completely and override selected helpers to add necessary adjustments
5479 * to their core operation.
5480 *
5481 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
5482 * any performance for debug and analysis features.
5483 *
5484 * @{
5485 */
5486
5487/**
5488 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
5489 * the debug run loop.
5490 */
5491typedef struct SVMRUNDBGSTATE
5492{
5493 /** The initial SVMVMCBCTRL::u64InterceptCtrl value (helps with restore). */
5494 uint64_t bmInterceptInitial;
5495 /** The initial SVMVMCBCTRL::u32InterceptXcpt value (helps with restore). */
5496 uint32_t bmXcptInitial;
5497 /** The initial SVMVMCBCTRL::u16InterceptRdCRx value (helps with restore). */
5498 uint16_t bmInterceptRdCRxInitial;
5499 /** The initial SVMVMCBCTRL::u16InterceptWrCRx value (helps with restore). */
5500 uint16_t bmInterceptWrCRxInitial;
5501 /** The initial SVMVMCBCTRL::u16InterceptRdDRx value (helps with restore). */
5502 uint16_t bmInterceptRdDRxInitial;
5503 /** The initial SVMVMCBCTRL::u16InterceptWrDRx value (helps with restore). */
5504 uint16_t bmInterceptWrDRxInitial;
5505
5506 /** Whether we've actually modified the intercept control qword. */
5507 bool fModifiedInterceptCtrl : 1;
5508 /** Whether we've actually modified the exception bitmap. */
5509 bool fModifiedXcptBitmap : 1;
5510 /** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdCRx. */
5511 bool fModifiedInterceptRdCRx : 1;
5512 /** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrCRx. */
5513 bool fModifiedInterceptWrCRx : 1;
5514 /** Whether we've actually modified SVMVMCBCTRL::u16InterceptRdDRx. */
5515 bool fModifiedInterceptRdDRx : 1;
5516 /** Whether we've actually modified SVMVMCBCTRL::u16InterceptWrDRx. */
5517 bool fModifiedInterceptWrDRx : 1;
5518
5519 /** The CS we started executing with. */
5520 uint16_t uCsStart;
5521 /** The RIP we started executing at. This is for detecting that we stepped. */
5522 uint64_t uRipStart;
5523
5524 /** The sequence number of the Dtrace provider settings the state was
5525 * configured against. */
5526 uint32_t uDtraceSettingsSeqNo;
5527 /** Extra stuff we need in SVMVMCBCTRL::u32InterceptXcpt. */
5528 uint32_t bmXcptExtra;
5529 /** Extra stuff we need in SVMVMCBCTRL::u64InterceptCtrl. */
5530 uint64_t bmInterceptExtra;
5531 /** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdCRx. */
5532 uint16_t bmInterceptRdCRxExtra;
5533 /** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrCRx. */
5534 uint16_t bmInterceptWrCRxExtra;
5535 /** Extra stuff we need in SVMVMCBCTRL::u16InterceptRdDRx. */
5536 uint16_t bmInterceptRdDRxExtra;
5537 /** Extra stuff we need in SVMVMCBCTRL::u16InterceptWrDRx. */
5538 uint16_t bmInterceptWrDRxExtra;
5539 /** VM-exits to check (one bit per VM-exit). */
5540 uint32_t bmExitsToCheck[33];
5541} SVMRUNDBGSTATE;
5542AssertCompileMemberSize(SVMRUNDBGSTATE, bmExitsToCheck, (SVM_EXIT_MAX + 1 + 31) / 32 * 4);
5543typedef SVMRUNDBGSTATE *PSVMRUNDBGSTATE;
5544
5545
5546/**
5547 * Initializes the SVMRUNDBGSTATE structure.
5548 *
5549 * @param pVCpu The cross context virtual CPU structure of the
5550 * calling EMT.
5551 * @param pSvmTransient The SVM-transient structure.
5552 * @param pDbgState The debug state to initialize.
5553 */
5554static void hmR0SvmRunDebugStateInit(PVMCPUCC pVCpu, PCSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5555{
5556 PSVMVMCB pVmcb = pSvmTransient->pVmcb;
5557 pDbgState->bmInterceptInitial = pVmcb->ctrl.u64InterceptCtrl;
5558 pDbgState->bmXcptInitial = pVmcb->ctrl.u32InterceptXcpt;
5559 pDbgState->bmInterceptRdCRxInitial = pVmcb->ctrl.u16InterceptRdCRx;
5560 pDbgState->bmInterceptWrCRxInitial = pVmcb->ctrl.u16InterceptWrCRx;
5561 pDbgState->bmInterceptRdDRxInitial = pVmcb->ctrl.u16InterceptRdDRx;
5562 pDbgState->bmInterceptWrDRxInitial = pVmcb->ctrl.u16InterceptWrDRx;
5563
5564 pDbgState->fModifiedInterceptCtrl = false;
5565 pDbgState->fModifiedXcptBitmap = false;
5566 pDbgState->fModifiedInterceptRdCRx = false;
5567 pDbgState->fModifiedInterceptWrCRx = false;
5568 pDbgState->fModifiedInterceptRdDRx = false;
5569 pDbgState->fModifiedInterceptWrDRx = false;
5570
5571 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
5572 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
5573
5574 /* We don't really need to zero these. */
5575 pDbgState->bmInterceptExtra = 0;
5576 pDbgState->bmXcptExtra = 0;
5577 pDbgState->bmInterceptRdCRxExtra = 0;
5578 pDbgState->bmInterceptWrCRxExtra = 0;
5579 pDbgState->bmInterceptRdDRxExtra = 0;
5580 pDbgState->bmInterceptWrDRxExtra = 0;
5581}
5582
5583
5584/**
5585 * Updates the VMCB fields with changes requested by @a pDbgState.
5586 *
5587 * This is performed after hmR0SvmPreRunGuestDebugStateUpdate as well
5588 * immediately before executing guest code, i.e. when interrupts are disabled.
5589 * We don't check status codes here as we cannot easily assert or return in the
5590 * latter case.
5591 *
5592 * @param pSvmTransient The SVM-transient structure.
5593 * @param pDbgState The debug state.
5594 */
5595static void hmR0SvmPreRunGuestDebugStateApply(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5596{
5597 /*
5598 * Ensure desired flags in VMCS control fields are set.
5599 */
5600 PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5601#define ADD_EXTRA_INTERCEPTS(a_VmcbCtrlField, a_bmExtra, a_fModified) do { \
5602 if ((pVmcb->ctrl. a_VmcbCtrlField & (a_bmExtra)) != (a_bmExtra)) \
5603 { \
5604 pVmcb->ctrl. a_VmcbCtrlField |= (a_bmExtra); \
5605 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5606 Log6Func((#a_VmcbCtrlField ": %#RX64\n", pVmcb->ctrl. a_VmcbCtrlField)); \
5607 (a_fModified) = true; \
5608 } \
5609 } while (0)
5610 ADD_EXTRA_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptExtra, pDbgState->fModifiedInterceptCtrl);
5611 ADD_EXTRA_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptExtra, pDbgState->fModifiedXcptBitmap);
5612 ADD_EXTRA_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxExtra, pDbgState->fModifiedInterceptRdCRx);
5613 ADD_EXTRA_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxExtra, pDbgState->fModifiedInterceptWrCRx);
5614 ADD_EXTRA_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxExtra, pDbgState->fModifiedInterceptRdDRx);
5615 ADD_EXTRA_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxExtra, pDbgState->fModifiedInterceptWrDRx);
5616#undef ADD_EXTRA_INTERCEPTS
5617}
5618
5619
5620/**
5621 * Restores VMCB fields that were changed by hmR0SvmPreRunGuestDebugStateApply
5622 * for re-entry next time around.
5623 *
5624 * @returns Strict VBox status code (i.e. informational status codes too).
5625 * @param pSvmTransient The SVM-transient structure.
5626 * @param pDbgState The debug state.
5627 */
5628static void hmR0SvmRunDebugStateRevert(PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5629{
5630 /*
5631 * Restore VM-exit control settings as we may not reenter this function the
5632 * next time around.
5633 */
5634 PSVMVMCB const pVmcb = pSvmTransient->pVmcb;
5635
5636#define RESTORE_INTERCEPTS(a_VmcbCtrlField, a_bmInitial, a_fModified) do { \
5637 if ((a_fModified)) \
5638 { \
5639 pVmcb->ctrl. a_VmcbCtrlField = (a_bmInitial); \
5640 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS; \
5641 } \
5642 } while (0)
5643 RESTORE_INTERCEPTS(u64InterceptCtrl, pDbgState->bmInterceptInitial, pDbgState->fModifiedInterceptCtrl);
5644 RESTORE_INTERCEPTS(u32InterceptXcpt, pDbgState->bmXcptInitial, pDbgState->fModifiedXcptBitmap);
5645 RESTORE_INTERCEPTS(u16InterceptRdCRx, pDbgState->bmInterceptRdCRxInitial, pDbgState->fModifiedInterceptRdCRx);
5646 RESTORE_INTERCEPTS(u16InterceptWrCRx, pDbgState->bmInterceptWrCRxInitial, pDbgState->fModifiedInterceptWrCRx);
5647 RESTORE_INTERCEPTS(u16InterceptRdDRx, pDbgState->bmInterceptRdDRxInitial, pDbgState->fModifiedInterceptRdDRx);
5648 RESTORE_INTERCEPTS(u16InterceptWrDRx, pDbgState->bmInterceptWrDRxInitial, pDbgState->fModifiedInterceptWrDRx);
5649#undef RESTORE_INTERCEPTS
5650}
5651
5652
5653/**
5654 * Configures VM-exit controls for current DBGF and DTrace settings.
5655 *
5656 * This updates @a pDbgState and the VMCB execution control fields (in the debug
5657 * state) to reflect the necessary VM-exits demanded by DBGF and DTrace.
5658 *
5659 * @param pVCpu The cross context virtual CPU structure.
5660 * @param pSvmTransient The SVM-transient structure. May update
5661 * fUpdatedTscOffsettingAndPreemptTimer.
5662 * @param pDbgState The debug state.
5663 */
5664static void hmR0SvmPreRunGuestDebugStateUpdate(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
5665{
5666 /*
5667 * Take down the dtrace serial number so we can spot changes.
5668 */
5669 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
5670 ASMCompilerBarrier();
5671
5672 /*
5673 * Clear data members that we'll be rebuilding here.
5674 */
5675 pDbgState->bmXcptExtra = 0;
5676 pDbgState->bmInterceptExtra = 0;
5677 pDbgState->bmInterceptRdCRxExtra = 0;
5678 pDbgState->bmInterceptWrCRxExtra = 0;
5679 pDbgState->bmInterceptRdDRxExtra = 0;
5680 pDbgState->bmInterceptWrDRxExtra = 0;
5681 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
5682 pDbgState->bmExitsToCheck[i] = 0;
5683
5684 /*
5685 * Software interrupts (INT XXh)
5686 */
5687 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5688 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
5689 || VBOXVMM_INT_SOFTWARE_ENABLED())
5690 {
5691 pDbgState->bmInterceptExtra |= SVM_CTRL_INTERCEPT_INTN;
5692 ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SWINT);
5693 }
5694
5695 /*
5696 * INT3 breakpoints - triggered by #BP exceptions.
5697 */
5698 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
5699 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
5700
5701 /*
5702 * Exception bitmap and XCPT events+probes.
5703 */
5704#define SET_XCPT(a_iXcpt) do { \
5705 pDbgState->bmXcptExtra |= RT_BIT_32(a_iXcpt); \
5706 ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_XCPT_0 + (a_iXcpt)); \
5707 } while (0)
5708
5709 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
5710 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
5711 SET_XCPT(iXcpt);
5712
5713 if (VBOXVMM_XCPT_DE_ENABLED()) SET_XCPT(X86_XCPT_DE);
5714 if (VBOXVMM_XCPT_DB_ENABLED()) SET_XCPT(X86_XCPT_DB);
5715 if (VBOXVMM_XCPT_BP_ENABLED()) SET_XCPT(X86_XCPT_BP);
5716 if (VBOXVMM_XCPT_OF_ENABLED()) SET_XCPT(X86_XCPT_OF);
5717 if (VBOXVMM_XCPT_BR_ENABLED()) SET_XCPT(X86_XCPT_BR);
5718 if (VBOXVMM_XCPT_UD_ENABLED()) SET_XCPT(X86_XCPT_UD);
5719 if (VBOXVMM_XCPT_NM_ENABLED()) SET_XCPT(X86_XCPT_NM);
5720 if (VBOXVMM_XCPT_DF_ENABLED()) SET_XCPT(X86_XCPT_DF);
5721 if (VBOXVMM_XCPT_TS_ENABLED()) SET_XCPT(X86_XCPT_TS);
5722 if (VBOXVMM_XCPT_NP_ENABLED()) SET_XCPT(X86_XCPT_NP);
5723 if (VBOXVMM_XCPT_SS_ENABLED()) SET_XCPT(X86_XCPT_SS);
5724 if (VBOXVMM_XCPT_GP_ENABLED()) SET_XCPT(X86_XCPT_GP);
5725 if (VBOXVMM_XCPT_PF_ENABLED()) SET_XCPT(X86_XCPT_PF);
5726 if (VBOXVMM_XCPT_MF_ENABLED()) SET_XCPT(X86_XCPT_MF);
5727 if (VBOXVMM_XCPT_AC_ENABLED()) SET_XCPT(X86_XCPT_AC);
5728 if (VBOXVMM_XCPT_XF_ENABLED()) SET_XCPT(X86_XCPT_XF);
5729 if (VBOXVMM_XCPT_VE_ENABLED()) SET_XCPT(X86_XCPT_VE);
5730 if (VBOXVMM_XCPT_SX_ENABLED()) SET_XCPT(X86_XCPT_SX);
5731
5732#undef SET_XCPT
5733
5734 /*
5735 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
5736 *
5737 * Note! This is the reverse of what hmR0SvmHandleExitDtraceEvents does.
5738 * So, when adding/changing/removing please don't forget to update it.
5739 *
5740 * Some of the macros are picking up local variables to save horizontal space,
5741 * (being able to see it in a table is the lesser evil here).
5742 */
5743#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
5744 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
5745 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
5746#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
5747 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5748 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5749 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5750 } else do { } while (0)
5751#define SET_INCP_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fInterceptCtrl) \
5752 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
5753 { \
5754 (pDbgState)->bmInterceptExtra |= (a_fInterceptCtrl); \
5755 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
5756 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
5757 } else do { } while (0)
5758
5759 /** @todo double check these */
5760 /** @todo Check what more AMD-V specific we can intercept. */
5761 //SET_INCP_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH, SVM_CTRL_INTERCEPT_TASK_SWITCH);
5762 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, SVM_EXIT_TASK_SWITCH);
5763 SET_INCP_XBM_IF_EITHER_EN(INSTR_VMM_CALL, SVM_EXIT_VMMCALL, SVM_CTRL_INTERCEPT_VMMCALL);
5764 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, SVM_EXIT_VMMCALL);
5765 SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMRUN, SVM_EXIT_VMRUN, SVM_CTRL_INTERCEPT_VMRUN);
5766 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMRUN, SVM_EXIT_VMRUN);
5767 SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMLOAD, SVM_EXIT_VMLOAD, SVM_CTRL_INTERCEPT_VMLOAD);
5768 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMLOAD, SVM_EXIT_VMLOAD);
5769 SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_VMSAVE, SVM_EXIT_VMSAVE, SVM_CTRL_INTERCEPT_VMSAVE);
5770 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_VMSAVE, SVM_EXIT_VMSAVE);
5771 SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_STGI, SVM_EXIT_STGI, SVM_CTRL_INTERCEPT_STGI);
5772 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_STGI, SVM_EXIT_STGI);
5773 SET_INCP_XBM_IF_EITHER_EN(INSTR_SVM_CLGI, SVM_EXIT_CLGI, SVM_CTRL_INTERCEPT_CLGI);
5774 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SVM_CLGI, SVM_EXIT_CLGI);
5775
5776 SET_INCP_XBM_IF_EITHER_EN(INSTR_CPUID, SVM_EXIT_CPUID, SVM_CTRL_INTERCEPT_CPUID);
5777 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, SVM_EXIT_CPUID);
5778 SET_INCP_XBM_IF_EITHER_EN(INSTR_HALT, SVM_EXIT_HLT, SVM_CTRL_INTERCEPT_HLT);
5779 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, SVM_EXIT_HLT);
5780 SET_INCP_XBM_IF_EITHER_EN(INSTR_INVD, SVM_EXIT_INVD, SVM_CTRL_INTERCEPT_INVD);
5781 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, SVM_EXIT_INVD);
5782 SET_INCP_XBM_IF_EITHER_EN(INSTR_INVLPG, SVM_EXIT_INVLPG, SVM_CTRL_INTERCEPT_INVLPG);
5783 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, SVM_EXIT_INVLPG);
5784 SET_INCP_XBM_IF_EITHER_EN(INSTR_RDPMC, SVM_EXIT_RDPMC, SVM_CTRL_INTERCEPT_RDPMC);
5785 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, SVM_EXIT_RDPMC);
5786 SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSC, SVM_EXIT_RDTSC, SVM_CTRL_INTERCEPT_RDTSC);
5787 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, SVM_EXIT_RDTSC);
5788 SET_INCP_XBM_IF_EITHER_EN(INSTR_RDTSCP, SVM_EXIT_RDTSCP, SVM_CTRL_INTERCEPT_RDTSCP);
5789 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, SVM_EXIT_RDTSCP);
5790 SET_INCP_XBM_IF_EITHER_EN(INSTR_RSM, SVM_EXIT_RSM, SVM_CTRL_INTERCEPT_RSM);
5791 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, SVM_EXIT_RSM);
5792
5793 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
5794 pDbgState->bmInterceptRdCRxExtra = 0xffff;
5795 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ) || IS_EITHER_ENABLED(pVM, EXIT_CRX_READ))
5796 ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_CR0, SVM_EXIT_READ_CR15 + 1);
5797
5798 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
5799 pDbgState->bmInterceptWrCRxExtra = 0xffff;
5800 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE) || IS_EITHER_ENABLED(pVM, EXIT_CRX_WRITE))
5801 {
5802 ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_CR0, SVM_EXIT_WRITE_CR15 + 1);
5803 ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_CR0_SEL_WRITE);
5804 }
5805
5806 if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ))
5807 pDbgState->bmInterceptRdDRxExtra = 0xffff;
5808 if (IS_EITHER_ENABLED(pVM, INSTR_DRX_READ) || IS_EITHER_ENABLED(pVM, EXIT_DRX_READ))
5809 ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_READ_DR0, SVM_EXIT_READ_DR15 + 1);
5810
5811 if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
5812 pDbgState->bmInterceptWrDRxExtra = 0xffff;
5813 if (IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE) || IS_EITHER_ENABLED(pVM, EXIT_DRX_WRITE))
5814 ASMBitSetRange(pDbgState->bmExitsToCheck, SVM_EXIT_WRITE_DR0, SVM_EXIT_WRITE_DR15 + 1);
5815
5816 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RDMSR, SVM_EXIT_MSR); /** @todo modify bitmap to intercept almost everything? (Clearing MSR_PROT just means no intercepts.) */
5817 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, SVM_EXIT_MSR);
5818 SET_ONLY_XBM_IF_EITHER_EN(INSTR_WRMSR, SVM_EXIT_MSR); /** @todo ditto */
5819 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, SVM_EXIT_MSR);
5820 SET_INCP_XBM_IF_EITHER_EN(INSTR_MWAIT, SVM_EXIT_MWAIT, SVM_CTRL_INTERCEPT_MWAIT);
5821 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, SVM_EXIT_MWAIT);
5822 if (ASMBitTest(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT))
5823 ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_MWAIT_ARMED);
5824 SET_INCP_XBM_IF_EITHER_EN(INSTR_MONITOR, SVM_EXIT_MONITOR, SVM_CTRL_INTERCEPT_MONITOR);
5825 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, SVM_EXIT_MONITOR);
5826 SET_INCP_XBM_IF_EITHER_EN(INSTR_PAUSE, SVM_EXIT_PAUSE, SVM_CTRL_INTERCEPT_PAUSE);
5827 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, SVM_EXIT_PAUSE);
5828 SET_INCP_XBM_IF_EITHER_EN(INSTR_SIDT, SVM_EXIT_IDTR_READ, SVM_CTRL_INTERCEPT_IDTR_READS);
5829 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, SVM_EXIT_IDTR_READ);
5830 SET_INCP_XBM_IF_EITHER_EN(INSTR_LIDT, SVM_EXIT_IDTR_WRITE, SVM_CTRL_INTERCEPT_IDTR_WRITES);
5831 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, SVM_EXIT_IDTR_WRITE);
5832 SET_INCP_XBM_IF_EITHER_EN(INSTR_SGDT, SVM_EXIT_GDTR_READ, SVM_CTRL_INTERCEPT_GDTR_READS);
5833 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, SVM_EXIT_GDTR_READ);
5834 SET_INCP_XBM_IF_EITHER_EN(INSTR_LGDT, SVM_EXIT_GDTR_WRITE, SVM_CTRL_INTERCEPT_GDTR_WRITES);
5835 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, SVM_EXIT_GDTR_WRITE);
5836 SET_INCP_XBM_IF_EITHER_EN(INSTR_SLDT, SVM_EXIT_LDTR_READ, SVM_CTRL_INTERCEPT_LDTR_READS);
5837 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, SVM_EXIT_LDTR_READ);
5838 SET_INCP_XBM_IF_EITHER_EN(INSTR_LLDT, SVM_EXIT_LDTR_WRITE, SVM_CTRL_INTERCEPT_LDTR_WRITES);
5839 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, SVM_EXIT_LDTR_WRITE);
5840 SET_INCP_XBM_IF_EITHER_EN(INSTR_STR, SVM_EXIT_TR_READ, SVM_CTRL_INTERCEPT_TR_READS);
5841 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, SVM_EXIT_TR_READ);
5842 SET_INCP_XBM_IF_EITHER_EN(INSTR_LTR, SVM_EXIT_TR_WRITE, SVM_CTRL_INTERCEPT_TR_WRITES);
5843 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, SVM_EXIT_TR_WRITE);
5844 SET_INCP_XBM_IF_EITHER_EN(INSTR_WBINVD, SVM_EXIT_WBINVD, SVM_CTRL_INTERCEPT_WBINVD);
5845 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, SVM_EXIT_WBINVD);
5846 SET_INCP_XBM_IF_EITHER_EN(INSTR_XSETBV, SVM_EXIT_XSETBV, SVM_CTRL_INTERCEPT_XSETBV);
5847 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, SVM_EXIT_XSETBV);
5848
5849 if (DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_TRIPLE_FAULT))
5850 ASMBitSet(pDbgState->bmExitsToCheck, SVM_EXIT_SHUTDOWN);
5851
5852#undef IS_EITHER_ENABLED
5853#undef SET_ONLY_XBM_IF_EITHER_EN
5854#undef SET_INCP_XBM_IF_EITHER_EN
5855
5856 /*
5857 * Sanitize the control stuff.
5858 */
5859 /** @todo filter out unsupported stuff? */
5860 if ( pVCpu->hmr0.s.fDebugWantRdTscExit
5861 != RT_BOOL(pDbgState->bmInterceptExtra & (SVM_CTRL_INTERCEPT_RDTSC | SVM_CTRL_INTERCEPT_RDTSCP)))
5862 {
5863 pVCpu->hmr0.s.fDebugWantRdTscExit ^= true;
5864 /// @todo pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
5865 RT_NOREF(pSvmTransient);
5866 }
5867
5868 Log6(("HM: debug state: bmInterceptExtra=%#RX64 bmXcptExtra=%#RX32%s%s%s%s bmExitsToCheck=%08RX32'%08RX32'%08RX32'%08RX32'%08RX32\n",
5869 pDbgState->bmInterceptExtra, pDbgState->bmXcptExtra,
5870 pDbgState->bmInterceptRdCRxExtra ? " rd-cr" : "",
5871 pDbgState->bmInterceptWrCRxExtra ? " wr-cr" : "",
5872 pDbgState->bmInterceptRdDRxExtra ? " rd-dr" : "",
5873 pDbgState->bmInterceptWrDRxExtra ? " wr-dr" : "",
5874 pDbgState->bmExitsToCheck[0],
5875 pDbgState->bmExitsToCheck[1],
5876 pDbgState->bmExitsToCheck[2],
5877 pDbgState->bmExitsToCheck[3],
5878 pDbgState->bmExitsToCheck[4]));
5879}
5880
5881
5882/**
5883 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
5884 * appropriate.
5885 *
5886 * The caller has checked the VM-exit against the SVMRUNDBGSTATE::bmExitsToCheck
5887 * bitmap.
5888 *
5889 * @returns Strict VBox status code (i.e. informational status codes too).
5890 * @param pVCpu The cross context virtual CPU structure.
5891 * @param pSvmTransient The SVM-transient structure.
5892 * @param uExitCode The VM-exit code.
5893 *
5894 * @remarks The name of this function is displayed by dtrace, so keep it short
5895 * and to the point. No longer than 33 chars long, please.
5896 */
5897static VBOXSTRICTRC hmR0SvmHandleExitDtraceEvents(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, uint64_t uExitCode)
5898{
5899 /*
5900 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
5901 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
5902 *
5903 * Note! This is the reverse operation of what hmR0SvmPreRunGuestDebugStateUpdate
5904 * does. Must add/change/remove both places. Same ordering, please.
5905 *
5906 * Added/removed events must also be reflected in the next section
5907 * where we dispatch dtrace events.
5908 */
5909 bool fDtrace1 = false;
5910 bool fDtrace2 = false;
5911 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
5912 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
5913 uint64_t uEventArg = 0;
5914#define SET_XCPT(a_XcptName) \
5915 do { \
5916 enmEvent2 = RT_CONCAT(DBGFEVENT_XCPT_, a_XcptName); \
5917 fDtrace2 = RT_CONCAT3(VBOXVMM_XCPT_, a_XcptName, _ENABLED)(); \
5918 } while (0)
5919#define SET_EXIT(a_EventSubName) \
5920 do { \
5921 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5922 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5923 } while (0)
5924#define SET_BOTH(a_EventSubName) \
5925 do { \
5926 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
5927 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
5928 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
5929 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
5930 } while (0)
5931 switch (uExitCode)
5932 {
5933 case SVM_EXIT_SWINT:
5934 enmEvent2 = DBGFEVENT_INTERRUPT_SOFTWARE;
5935 fDtrace2 = VBOXVMM_INT_SOFTWARE_ENABLED();
5936 uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1;
5937 break;
5938
5939 case SVM_EXIT_XCPT_DE: SET_XCPT(DE); break;
5940 case SVM_EXIT_XCPT_DB: SET_XCPT(DB); break;
5941 case SVM_EXIT_XCPT_BP: SET_XCPT(BP); break;
5942 case SVM_EXIT_XCPT_OF: SET_XCPT(OF); break;
5943 case SVM_EXIT_XCPT_BR: SET_XCPT(BR); break;
5944 case SVM_EXIT_XCPT_UD: SET_XCPT(UD); break;
5945 case SVM_EXIT_XCPT_NM: SET_XCPT(NM); break;
5946 case SVM_EXIT_XCPT_DF: SET_XCPT(DF); break;
5947 case SVM_EXIT_XCPT_TS: SET_XCPT(TS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5948 case SVM_EXIT_XCPT_NP: SET_XCPT(NP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5949 case SVM_EXIT_XCPT_SS: SET_XCPT(SS); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5950 case SVM_EXIT_XCPT_GP: SET_XCPT(GP); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5951 case SVM_EXIT_XCPT_PF: SET_XCPT(PF); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5952 case SVM_EXIT_XCPT_MF: SET_XCPT(MF); break;
5953 case SVM_EXIT_XCPT_AC: SET_XCPT(AC); break;
5954 case SVM_EXIT_XCPT_XF: SET_XCPT(XF); break;
5955 case SVM_EXIT_XCPT_VE: SET_XCPT(VE); break;
5956 case SVM_EXIT_XCPT_SX: SET_XCPT(SX); uEventArg = pSvmTransient->pVmcb->ctrl.u64ExitInfo1; break;
5957
5958 case SVM_EXIT_XCPT_2: enmEvent2 = DBGFEVENT_XCPT_02; break;
5959 case SVM_EXIT_XCPT_9: enmEvent2 = DBGFEVENT_XCPT_09; break;
5960 case SVM_EXIT_XCPT_15: enmEvent2 = DBGFEVENT_XCPT_0f; break;
5961 case SVM_EXIT_XCPT_18: enmEvent2 = DBGFEVENT_XCPT_MC; break;
5962 case SVM_EXIT_XCPT_21: enmEvent2 = DBGFEVENT_XCPT_15; break;
5963 case SVM_EXIT_XCPT_22: enmEvent2 = DBGFEVENT_XCPT_16; break;
5964 case SVM_EXIT_XCPT_23: enmEvent2 = DBGFEVENT_XCPT_17; break;
5965 case SVM_EXIT_XCPT_24: enmEvent2 = DBGFEVENT_XCPT_18; break;
5966 case SVM_EXIT_XCPT_25: enmEvent2 = DBGFEVENT_XCPT_19; break;
5967 case SVM_EXIT_XCPT_26: enmEvent2 = DBGFEVENT_XCPT_1a; break;
5968 case SVM_EXIT_XCPT_27: enmEvent2 = DBGFEVENT_XCPT_1b; break;
5969 case SVM_EXIT_XCPT_28: enmEvent2 = DBGFEVENT_XCPT_1c; break;
5970 case SVM_EXIT_XCPT_29: enmEvent2 = DBGFEVENT_XCPT_1d; break;
5971 case SVM_EXIT_XCPT_31: enmEvent2 = DBGFEVENT_XCPT_1f; break;
5972
5973 case SVM_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
5974 case SVM_EXIT_VMMCALL: SET_BOTH(VMM_CALL); break;
5975 case SVM_EXIT_VMRUN: SET_BOTH(SVM_VMRUN); break;
5976 case SVM_EXIT_VMLOAD: SET_BOTH(SVM_VMLOAD); break;
5977 case SVM_EXIT_VMSAVE: SET_BOTH(SVM_VMSAVE); break;
5978 case SVM_EXIT_STGI: SET_BOTH(SVM_STGI); break;
5979 case SVM_EXIT_CLGI: SET_BOTH(SVM_CLGI); break;
5980 case SVM_EXIT_CPUID: SET_BOTH(CPUID); break;
5981 case SVM_EXIT_HLT: SET_BOTH(HALT); break;
5982 case SVM_EXIT_INVD: SET_BOTH(INVD); break;
5983 case SVM_EXIT_INVLPG: SET_BOTH(INVLPG); break;
5984 case SVM_EXIT_RDPMC: SET_BOTH(RDPMC); break;
5985 case SVM_EXIT_RDTSC: SET_BOTH(RDTSC); break;
5986 case SVM_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
5987 case SVM_EXIT_RSM: SET_BOTH(RSM); break;
5988
5989 case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
5990 case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
5991 case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
5992 case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
5993 SET_BOTH(CRX_READ);
5994 uEventArg = uExitCode - SVM_EXIT_READ_CR0;
5995 break;
5996 case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
5997 case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
5998 case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
5999 case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
6000 case SVM_EXIT_CR0_SEL_WRITE:
6001 SET_BOTH(CRX_WRITE);
6002 uEventArg = uExitCode - SVM_EXIT_WRITE_CR0;
6003 break;
6004 case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6005 case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6006 case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6007 case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6008 SET_BOTH(DRX_READ);
6009 uEventArg = uExitCode - SVM_EXIT_READ_DR0;
6010 break;
6011 case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6012 case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6013 case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6014 case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6015 SET_BOTH(DRX_WRITE);
6016 uEventArg = uExitCode - SVM_EXIT_WRITE_DR0;
6017 break;
6018 case SVM_EXIT_MSR:
6019 if (pSvmTransient->pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6020 SET_BOTH(WRMSR);
6021 else
6022 SET_BOTH(RDMSR);
6023 break;
6024 case SVM_EXIT_MWAIT_ARMED:
6025 case SVM_EXIT_MWAIT: SET_BOTH(MWAIT); break;
6026 case SVM_EXIT_MONITOR: SET_BOTH(MONITOR); break;
6027 case SVM_EXIT_PAUSE: SET_BOTH(PAUSE); break;
6028 case SVM_EXIT_IDTR_READ: SET_BOTH(SIDT); break;
6029 case SVM_EXIT_IDTR_WRITE: SET_BOTH(LIDT); break;
6030 case SVM_EXIT_GDTR_READ: SET_BOTH(SGDT); break;
6031 case SVM_EXIT_GDTR_WRITE: SET_BOTH(LGDT); break;
6032 case SVM_EXIT_LDTR_READ: SET_BOTH(SLDT); break;
6033 case SVM_EXIT_LDTR_WRITE: SET_BOTH(LLDT); break;
6034 case SVM_EXIT_TR_READ: SET_BOTH(STR); break;
6035 case SVM_EXIT_TR_WRITE: SET_BOTH(LTR); break;
6036 case SVM_EXIT_WBINVD: SET_BOTH(WBINVD); break;
6037 case SVM_EXIT_XSETBV: SET_BOTH(XSETBV); break;
6038
6039 case SVM_EXIT_SHUTDOWN:
6040 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
6041 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
6042 break;
6043
6044 default:
6045 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6046 break;
6047 }
6048#undef SET_BOTH
6049#undef SET_EXIT
6050
6051 /*
6052 * Dtrace tracepoints go first. We do them here at once so we don't
6053 * have to copy the guest state saving and stuff a few dozen times.
6054 * Down side is that we've got to repeat the switch, though this time
6055 * we use enmEvent since the probes are a subset of what DBGF does.
6056 */
6057 if (fDtrace1 || fDtrace2)
6058 {
6059 hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6060 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6061 switch (enmEvent1)
6062 {
6063 /** @todo consider which extra parameters would be helpful for each probe. */
6064 case DBGFEVENT_END: break;
6065 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6066 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
6067 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
6068 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
6069 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
6070 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
6071 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
6072 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
6073 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
6074 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6075 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6076 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, (uint32_t)uEventArg); break;
6077 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, (uint32_t)uEventArg); break;
6078 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, (uint32_t)uEventArg, pCtx->cr2); break;
6079 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
6080 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
6081 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
6082 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
6083 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, (uint32_t)uEventArg); break;
6084 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6085 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
6086 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
6087 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
6088 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
6089 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
6090 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
6091 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6092 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6093 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6094 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6095 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6096 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
6097 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6098 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
6099 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
6100 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
6101 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
6102 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
6103 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
6104 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
6105 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
6106 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
6107 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
6108 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
6109 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
6110 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
6111 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
6112 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
6113 case DBGFEVENT_INSTR_SVM_VMRUN: VBOXVMM_INSTR_SVM_VMRUN(pVCpu, pCtx); break;
6114 case DBGFEVENT_INSTR_SVM_VMLOAD: VBOXVMM_INSTR_SVM_VMLOAD(pVCpu, pCtx); break;
6115 case DBGFEVENT_INSTR_SVM_VMSAVE: VBOXVMM_INSTR_SVM_VMSAVE(pVCpu, pCtx); break;
6116 case DBGFEVENT_INSTR_SVM_STGI: VBOXVMM_INSTR_SVM_STGI(pVCpu, pCtx); break;
6117 case DBGFEVENT_INSTR_SVM_CLGI: VBOXVMM_INSTR_SVM_CLGI(pVCpu, pCtx); break;
6118 default: AssertMsgFailed(("enmEvent1=%d uExitCode=%d\n", enmEvent1, uExitCode)); break;
6119 }
6120 switch (enmEvent2)
6121 {
6122 /** @todo consider which extra parameters would be helpful for each probe. */
6123 case DBGFEVENT_END: break;
6124 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
6125 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
6126 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
6127 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
6128 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
6129 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
6130 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
6131 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
6132 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6133 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6134 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
6135 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
6136 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
6137 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
6138 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
6139 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
6140 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
6141 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
6142 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
6143 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
6144 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
6145 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
6146 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
6147 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
6148 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
6149 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
6150 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
6151 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
6152 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
6153 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
6154 case DBGFEVENT_EXIT_SVM_VMRUN: VBOXVMM_EXIT_SVM_VMRUN(pVCpu, pCtx); break;
6155 case DBGFEVENT_EXIT_SVM_VMLOAD: VBOXVMM_EXIT_SVM_VMLOAD(pVCpu, pCtx); break;
6156 case DBGFEVENT_EXIT_SVM_VMSAVE: VBOXVMM_EXIT_SVM_VMSAVE(pVCpu, pCtx); break;
6157 case DBGFEVENT_EXIT_SVM_STGI: VBOXVMM_EXIT_SVM_STGI(pVCpu, pCtx); break;
6158 case DBGFEVENT_EXIT_SVM_CLGI: VBOXVMM_EXIT_SVM_CLGI(pVCpu, pCtx); break;
6159 default: AssertMsgFailed(("enmEvent2=%d uExitCode=%d\n", enmEvent2, uExitCode)); break;
6160 }
6161 }
6162
6163 /*
6164 * Fire of the DBGF event, if enabled (our check here is just a quick one,
6165 * the DBGF call will do a full check).
6166 *
6167 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
6168 * Note! If we have to events, we prioritize the first, i.e. the instruction
6169 * one, in order to avoid event nesting.
6170 */
6171 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
6172 VBOXSTRICTRC rcStrict;
6173 if ( enmEvent1 != DBGFEVENT_END
6174 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
6175 {
6176 hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
6177 rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
6178 }
6179 else if ( enmEvent2 != DBGFEVENT_END
6180 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
6181 {
6182 hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
6183 rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
6184 }
6185 else
6186 rcStrict = VINF_SUCCESS;
6187 return rcStrict;
6188}
6189
6190
6191/**
6192 * Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID),
6193 * debug variant.
6194 *
6195 * @returns Strict VBox status code (informational status codes included).
6196 * @param pVCpu The cross context virtual CPU structure.
6197 * @param pSvmTransient Pointer to the SVM transient structure.
6198 * @param pDbgState The runtime debug state.
6199 */
6200static VBOXSTRICTRC hmR0SvmDebugHandleExit(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient, PSVMRUNDBGSTATE pDbgState)
6201{
6202 Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
6203 Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
6204
6205 /*
6206 * Expensive (saves context) generic dtrace VM-exit probe.
6207 */
6208 uint64_t const uExitCode = pSvmTransient->u64ExitCode;
6209 if (!VBOXVMM_R0_HMSVM_VMEXIT_ENABLED())
6210 { /* more likely */ }
6211 else
6212 {
6213 hmR0SvmImportGuestState(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
6214 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, uExitCode, pSvmTransient->pVmcb);
6215 }
6216
6217 /*
6218 * Check for single stepping event if we're stepping.
6219 */
6220 if (pVCpu->hm.s.fSingleInstruction)
6221 {
6222 switch (uExitCode)
6223 {
6224 /* Various events: */
6225 case SVM_EXIT_XCPT_0: case SVM_EXIT_XCPT_1: case SVM_EXIT_XCPT_2: case SVM_EXIT_XCPT_3:
6226 case SVM_EXIT_XCPT_4: case SVM_EXIT_XCPT_5: case SVM_EXIT_XCPT_6: case SVM_EXIT_XCPT_7:
6227 case SVM_EXIT_XCPT_8: case SVM_EXIT_XCPT_9: case SVM_EXIT_XCPT_10: case SVM_EXIT_XCPT_11:
6228 case SVM_EXIT_XCPT_12: case SVM_EXIT_XCPT_13: case SVM_EXIT_XCPT_14: case SVM_EXIT_XCPT_15:
6229 case SVM_EXIT_XCPT_16: case SVM_EXIT_XCPT_17: case SVM_EXIT_XCPT_18: case SVM_EXIT_XCPT_19:
6230 case SVM_EXIT_XCPT_20: case SVM_EXIT_XCPT_21: case SVM_EXIT_XCPT_22: case SVM_EXIT_XCPT_23:
6231 case SVM_EXIT_XCPT_24: case SVM_EXIT_XCPT_25: case SVM_EXIT_XCPT_26: case SVM_EXIT_XCPT_27:
6232 case SVM_EXIT_XCPT_28: case SVM_EXIT_XCPT_29: case SVM_EXIT_XCPT_30: case SVM_EXIT_XCPT_31:
6233 case SVM_EXIT_INTR:
6234 case SVM_EXIT_NMI:
6235 case SVM_EXIT_VINTR:
6236 case SVM_EXIT_NPF:
6237 case SVM_EXIT_AVIC_NOACCEL:
6238
6239 /* Instruction specific VM-exits: */
6240 case SVM_EXIT_READ_CR0: case SVM_EXIT_READ_CR1: case SVM_EXIT_READ_CR2: case SVM_EXIT_READ_CR3:
6241 case SVM_EXIT_READ_CR4: case SVM_EXIT_READ_CR5: case SVM_EXIT_READ_CR6: case SVM_EXIT_READ_CR7:
6242 case SVM_EXIT_READ_CR8: case SVM_EXIT_READ_CR9: case SVM_EXIT_READ_CR10: case SVM_EXIT_READ_CR11:
6243 case SVM_EXIT_READ_CR12: case SVM_EXIT_READ_CR13: case SVM_EXIT_READ_CR14: case SVM_EXIT_READ_CR15:
6244 case SVM_EXIT_WRITE_CR0: case SVM_EXIT_WRITE_CR1: case SVM_EXIT_WRITE_CR2: case SVM_EXIT_WRITE_CR3:
6245 case SVM_EXIT_WRITE_CR4: case SVM_EXIT_WRITE_CR5: case SVM_EXIT_WRITE_CR6: case SVM_EXIT_WRITE_CR7:
6246 case SVM_EXIT_WRITE_CR8: case SVM_EXIT_WRITE_CR9: case SVM_EXIT_WRITE_CR10: case SVM_EXIT_WRITE_CR11:
6247 case SVM_EXIT_WRITE_CR12: case SVM_EXIT_WRITE_CR13: case SVM_EXIT_WRITE_CR14: case SVM_EXIT_WRITE_CR15:
6248 case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
6249 case SVM_EXIT_READ_DR4: case SVM_EXIT_READ_DR5: case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7:
6250 case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9: case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11:
6251 case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13: case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
6252 case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
6253 case SVM_EXIT_WRITE_DR4: case SVM_EXIT_WRITE_DR5: case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7:
6254 case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9: case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11:
6255 case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13: case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
6256 case SVM_EXIT_CR0_SEL_WRITE:
6257 case SVM_EXIT_IDTR_READ:
6258 case SVM_EXIT_GDTR_READ:
6259 case SVM_EXIT_LDTR_READ:
6260 case SVM_EXIT_TR_READ:
6261 case SVM_EXIT_IDTR_WRITE:
6262 case SVM_EXIT_GDTR_WRITE:
6263 case SVM_EXIT_LDTR_WRITE:
6264 case SVM_EXIT_TR_WRITE:
6265 case SVM_EXIT_RDTSC:
6266 case SVM_EXIT_RDPMC:
6267 case SVM_EXIT_PUSHF:
6268 case SVM_EXIT_POPF:
6269 case SVM_EXIT_CPUID:
6270 case SVM_EXIT_RSM:
6271 case SVM_EXIT_IRET:
6272 case SVM_EXIT_SWINT:
6273 case SVM_EXIT_INVD:
6274 case SVM_EXIT_PAUSE:
6275 case SVM_EXIT_HLT:
6276 case SVM_EXIT_INVLPG:
6277 case SVM_EXIT_INVLPGA:
6278 case SVM_EXIT_IOIO:
6279 case SVM_EXIT_MSR:
6280 case SVM_EXIT_TASK_SWITCH:
6281 case SVM_EXIT_VMRUN:
6282 case SVM_EXIT_VMMCALL:
6283 case SVM_EXIT_VMLOAD:
6284 case SVM_EXIT_VMSAVE:
6285 case SVM_EXIT_STGI:
6286 case SVM_EXIT_CLGI:
6287 case SVM_EXIT_SKINIT:
6288 case SVM_EXIT_RDTSCP:
6289 case SVM_EXIT_ICEBP:
6290 case SVM_EXIT_WBINVD:
6291 case SVM_EXIT_MONITOR:
6292 case SVM_EXIT_MWAIT:
6293 case SVM_EXIT_MWAIT_ARMED:
6294 case SVM_EXIT_XSETBV:
6295 case SVM_EXIT_RDPRU:
6296 case SVM_EXIT_WRITE_EFER_TRAP:
6297 case SVM_EXIT_WRITE_CR0_TRAP: case SVM_EXIT_WRITE_CR1_TRAP: case SVM_EXIT_WRITE_CR2_TRAP: case SVM_EXIT_WRITE_CR3_TRAP:
6298 case SVM_EXIT_WRITE_CR4_TRAP: case SVM_EXIT_WRITE_CR5_TRAP: case SVM_EXIT_WRITE_CR6_TRAP: case SVM_EXIT_WRITE_CR7_TRAP:
6299 case SVM_EXIT_WRITE_CR8_TRAP: case SVM_EXIT_WRITE_CR9_TRAP: case SVM_EXIT_WRITE_CR10_TRAP: case SVM_EXIT_WRITE_CR11_TRAP:
6300 case SVM_EXIT_WRITE_CR12_TRAP: case SVM_EXIT_WRITE_CR13_TRAP: case SVM_EXIT_WRITE_CR14_TRAP: case SVM_EXIT_WRITE_CR15_TRAP:
6301 case SVM_EXIT_MCOMMIT:
6302 {
6303 hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
6304 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
6305 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
6306 {
6307 Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6308 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode));
6309 return VINF_EM_DBG_STEPPED;
6310 }
6311 break;
6312 }
6313
6314 /* Errors and unexpected events: */
6315 case SVM_EXIT_FERR_FREEZE:
6316 case SVM_EXIT_SHUTDOWN:
6317 case SVM_EXIT_AVIC_INCOMPLETE_IPI:
6318 break;
6319
6320 case SVM_EXIT_SMI:
6321 case SVM_EXIT_INIT:
6322 default:
6323 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitCode));
6324 break;
6325 }
6326 }
6327
6328 /*
6329 * Check for debugger event breakpoints and dtrace probes.
6330 */
6331 if ( uExitCode < sizeof(pDbgState->bmExitsToCheck) * 8U
6332 && ASMBitTest(pDbgState->bmExitsToCheck, uExitCode) )
6333 {
6334 VBOXSTRICTRC rcStrict = hmR0SvmHandleExitDtraceEvents(pVCpu, pSvmTransient, uExitCode);
6335 if (rcStrict != VINF_SUCCESS)
6336 {
6337 Log6Func(("%04x:%08RX64 (exit %u) -> %Rrc\n",
6338 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, uExitCode, VBOXSTRICTRC_VAL(rcStrict) ));
6339 return rcStrict;
6340 }
6341 }
6342
6343 /*
6344 * Normal processing.
6345 */
6346 return hmR0SvmHandleExit(pVCpu, pSvmTransient);
6347}
6348
6349
6350/**
6351 * Runs the guest code using AMD-V in single step mode.
6352 *
6353 * @returns Strict VBox status code.
6354 * @param pVCpu The cross context virtual CPU structure.
6355 * @param pcLoops Pointer to the number of executed loops.
6356 */
6357static VBOXSTRICTRC hmR0SvmRunGuestCodeDebug(PVMCPUCC pVCpu, uint32_t *pcLoops)
6358{
6359 uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hmr0.s.cMaxResumeLoops;
6360 Assert(pcLoops);
6361 Assert(*pcLoops <= cMaxResumeLoops);
6362
6363 SVMTRANSIENT SvmTransient;
6364 RT_ZERO(SvmTransient);
6365 SvmTransient.fUpdateTscOffsetting = true;
6366 SvmTransient.pVmcb = pVCpu->hmr0.s.svm.pVmcb;
6367
6368 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6369
6370 /* Set HMCPU indicators. */
6371 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
6372 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
6373 pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6374 pVCpu->hmr0.s.fUsingDebugLoop = true;
6375
6376 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
6377 SVMRUNDBGSTATE DbgState;
6378 hmR0SvmRunDebugStateInit(pVCpu, &SvmTransient, &DbgState);
6379 hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6380
6381 /*
6382 * The loop.
6383 */
6384 VBOXSTRICTRC rc = VERR_INTERNAL_ERROR_5;
6385 for (;;)
6386 {
6387 Assert(!HMR0SuspendPending());
6388 AssertMsg(pVCpu->hmr0.s.idEnteredCpu == RTMpCpuId(),
6389 ("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hmr0.s.idEnteredCpu,
6390 (unsigned)RTMpCpuId(), *pcLoops));
6391 bool fStepping = pVCpu->hm.s.fSingleInstruction;
6392
6393 /* Set up VM-execution controls the next two can respond to. */
6394 hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6395
6396 /* Preparatory work for running nested-guest code, this may force us to return to
6397 ring-3. This bugger disables interrupts on VINF_SUCCESS! */
6398 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6399 rc = hmR0SvmPreRunGuest(pVCpu, &SvmTransient);
6400 if (rc != VINF_SUCCESS)
6401 break;
6402
6403 /*
6404 * No longjmps to ring-3 from this point on!!!
6405 *
6406 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional,
6407 * better than a kernel panic. This also disables flushing of the R0-logger instance.
6408 */
6409 hmR0SvmPreRunGuestCommitted(pVCpu, &SvmTransient);
6410
6411 /* Override any obnoxious code in the above two calls. */
6412 hmR0SvmPreRunGuestDebugStateApply(&SvmTransient, &DbgState);
6413#if 0
6414 Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64\n",
6415 SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6416 SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u));
6417#endif
6418
6419 /*
6420 * Finally execute guest code.
6421 */
6422 rc = hmR0SvmRunGuest(pVCpu, pVCpu->hmr0.s.svm.HCPhysVmcb);
6423
6424 /* Restore any residual host-state and save any bits shared between host and guest
6425 into the guest-CPU state. Re-enables interrupts! */
6426 hmR0SvmPostRunGuest(pVCpu, &SvmTransient, rc);
6427#if 0
6428 Log(("%04x:%08RX64 ds=%04x %04x:%08RX64 i=%#RX64 exit=%d\n",
6429 SvmTransient.pVmcb->guest.CS.u16Sel, SvmTransient.pVmcb->guest.u64RIP, SvmTransient.pVmcb->guest.DS.u16Sel,
6430 SvmTransient.pVmcb->guest.SS.u16Sel, SvmTransient.pVmcb->guest.u64RSP, SvmTransient.pVmcb->ctrl.EventInject.u, SvmTransient.u64ExitCode));
6431#endif
6432
6433 if (RT_LIKELY( rc == VINF_SUCCESS /* Check for VMRUN errors. */
6434 && SvmTransient.u64ExitCode != SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
6435 { /* very likely*/ }
6436 else
6437 {
6438 if (rc == VINF_SUCCESS)
6439 rc = VERR_SVM_INVALID_GUEST_STATE;
6440 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
6441 hmR0SvmReportWorldSwitchError(pVCpu, VBOXSTRICTRC_VAL(rc));
6442 return rc;
6443 }
6444
6445 /* Handle the #VMEXIT. */
6446 HMSVM_DEBUG_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
6447 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
6448 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hmr0.s.svm.pVmcb);
6449 rc = hmR0SvmDebugHandleExit(pVCpu, &SvmTransient, &DbgState);
6450 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
6451 if (rc != VINF_SUCCESS)
6452 break;
6453 if (++(*pcLoops) >= cMaxResumeLoops)
6454 {
6455 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
6456 rc = VINF_EM_RAW_INTERRUPT;
6457 break;
6458 }
6459
6460 /*
6461 * Stepping: Did the RIP change, if so, consider it a single step.
6462 * Otherwise, make sure one of the TFs gets set.
6463 */
6464 if (fStepping)
6465 {
6466 hmR0SvmImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
6467 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
6468 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
6469 {
6470 Log6Func(("VINF_EM_DBG_STEPPED: %04x:%08RX64 (exit %u)\n",
6471 pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, SvmTransient.u64ExitCode));
6472 rc = VINF_EM_DBG_STEPPED;
6473 break;
6474 }
6475 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
6476 }
6477
6478 /*
6479 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
6480 * Revert the state changes afterware so we can drop intercepts no longer needed.
6481 */
6482 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
6483 {
6484 hmR0SvmPreRunGuestDebugStateUpdate(pVCpu, &SvmTransient, &DbgState);
6485 hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6486 }
6487 }
6488
6489 /*
6490 * Clear the X86_EFL_TF if necessary.
6491 */
6492 if (pVCpu->hmr0.s.fClearTrapFlag)
6493 {
6494 pVCpu->hmr0.s.fClearTrapFlag = false;
6495 pCtx->eflags.Bits.u1TF = 0;
6496 }
6497
6498 /* Restore HMCPU indicators. */
6499 pVCpu->hmr0.s.fUsingDebugLoop = false;
6500 pVCpu->hmr0.s.fDebugWantRdTscExit = false;
6501 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
6502
6503 /* Restore all controls applied by hmR0SvmPreRunGuestDebugStateApply above. */
6504 hmR0SvmRunDebugStateRevert(&SvmTransient, &DbgState);
6505
6506 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6507 return rc;
6508}
6509
6510/** @} */
6511
6512#undef VMEXIT_CALL_RET
6513
6514
6515#ifdef VBOX_STRICT
6516/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6517# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
6518 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6519
6520# define HMSVM_ASSERT_PREEMPT_CPUID() \
6521 do \
6522 { \
6523 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6524 AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6525 } while (0)
6526
6527# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6528 do { \
6529 AssertPtr((a_pVCpu)); \
6530 AssertPtr((a_pSvmTransient)); \
6531 Assert(ASMIntAreEnabled()); \
6532 HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6533 HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
6534 Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (a_pVCpu)->idCpu)); \
6535 HMSVM_ASSERT_PREEMPT_SAFE((a_pVCpu)); \
6536 if (!VMMRZCallRing3IsEnabled((a_pVCpu))) \
6537 HMSVM_ASSERT_PREEMPT_CPUID(); \
6538 } while (0)
6539#else
6540# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pSvmTransient) \
6541 do { \
6542 RT_NOREF2(a_pVCpu, a_pSvmTransient); \
6543 } while (0)
6544#endif
6545
6546
6547/**
6548 * Gets the IEM exception flags for the specified SVM event.
6549 *
6550 * @returns The IEM exception flags.
6551 * @param pEvent Pointer to the SVM event.
6552 *
6553 * @remarks This function currently only constructs flags required for
6554 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
6555 * and CR2 aspects of an exception are not included).
6556 */
6557static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
6558{
6559 uint8_t const uEventType = pEvent->n.u3Type;
6560 uint32_t fIemXcptFlags;
6561 switch (uEventType)
6562 {
6563 case SVM_EVENT_EXCEPTION:
6564 /*
6565 * Only INT3 and INTO instructions can raise #BP and #OF exceptions.
6566 * See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
6567 */
6568 if (pEvent->n.u8Vector == X86_XCPT_BP)
6569 {
6570 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_BP_INSTR;
6571 break;
6572 }
6573 if (pEvent->n.u8Vector == X86_XCPT_OF)
6574 {
6575 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_OF_INSTR;
6576 break;
6577 }
6578 /** @todo How do we distinguish ICEBP \#DB from the regular one? */
6579 RT_FALL_THRU();
6580 case SVM_EVENT_NMI:
6581 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6582 break;
6583
6584 case SVM_EVENT_EXTERNAL_IRQ:
6585 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6586 break;
6587
6588 case SVM_EVENT_SOFTWARE_INT:
6589 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6590 break;
6591
6592 default:
6593 fIemXcptFlags = 0;
6594 AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
6595 break;
6596 }
6597 return fIemXcptFlags;
6598}
6599
6600
6601/**
6602 * Handle a condition that occurred while delivering an event through the guest
6603 * IDT.
6604 *
6605 * @returns VBox status code (informational error codes included).
6606 * @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
6607 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
6608 * continue execution of the guest which will delivery the \#DF.
6609 * @retval VINF_EM_RESET if we detected a triple-fault condition.
6610 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
6611 *
6612 * @param pVCpu The cross context virtual CPU structure.
6613 * @param pSvmTransient Pointer to the SVM transient structure.
6614 *
6615 * @remarks No-long-jump zone!!!
6616 */
6617static int hmR0SvmCheckExitDueToEventDelivery(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6618{
6619 /** @todo r=bird: Looks like this is called on many exits and we start by
6620 * loading CR2 on the offchance that we actually have work to do here.
6621 *
6622 * HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY can surely check
6623 * pVmcb->ctrl.ExitIntInfo.n.u1Valid, can't it?
6624 *
6625 * Also, what's the deal with hmR0SvmGetCurrentVmcb() vs pSvmTransient->pVmcb?
6626 */
6627 int rc = VINF_SUCCESS;
6628 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6629 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR2);
6630
6631 Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
6632 pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
6633 pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
6634
6635 /*
6636 * The EXITINTINFO (if valid) contains the prior exception (IDT vector) that was trying to
6637 * be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector).
6638 *
6639 * See AMD spec. 15.7.3 "EXITINFO Pseudo-Code".
6640 */
6641 if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
6642 {
6643 IEMXCPTRAISE enmRaise;
6644 IEMXCPTRAISEINFO fRaiseInfo;
6645 bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0 <= SVM_EXIT_XCPT_31;
6646 uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
6647 if (fExitIsHwXcpt)
6648 {
6649 uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_XCPT_0;
6650 uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
6651 uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6652 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
6653 }
6654 else
6655 {
6656 /*
6657 * If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF)
6658 * then we end up here.
6659 *
6660 * If the event was:
6661 * - a software interrupt, we can re-execute the instruction which will
6662 * regenerate the event.
6663 * - an NMI, we need to clear NMI blocking and re-inject the NMI.
6664 * - a hardware exception or external interrupt, we re-inject it.
6665 */
6666 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
6667 if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
6668 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
6669 else
6670 enmRaise = IEMXCPTRAISE_PREV_EVENT;
6671 }
6672
6673 switch (enmRaise)
6674 {
6675 case IEMXCPTRAISE_CURRENT_XCPT:
6676 case IEMXCPTRAISE_PREV_EVENT:
6677 {
6678 /* For software interrupts, we shall re-execute the instruction. */
6679 if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
6680 {
6681 RTGCUINTPTR GCPtrFaultAddress = 0;
6682
6683 /* If we are re-injecting an NMI, clear NMI blocking. */
6684 if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
6685 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6686
6687 /* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
6688 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
6689 {
6690 pSvmTransient->fVectoringPF = true;
6691 Log4Func(("IDT: Pending vectoring #PF due to delivery of Ext-Int/NMI. uCR2=%#RX64\n",
6692 pVCpu->cpum.GstCtx.cr2));
6693 }
6694 else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
6695 && uIdtVector == X86_XCPT_PF)
6696 {
6697 /*
6698 * If the previous exception was a #PF, we need to recover the CR2 value.
6699 * This can't happen with shadow paging.
6700 */
6701 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
6702 }
6703
6704 /*
6705 * Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
6706 * exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
6707 */
6708 Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
6709 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflect);
6710 hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
6711
6712 Log4Func(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
6713 pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
6714 pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
6715 }
6716 break;
6717 }
6718
6719 case IEMXCPTRAISE_REEXEC_INSTR:
6720 {
6721 Assert(rc == VINF_SUCCESS);
6722 break;
6723 }
6724
6725 case IEMXCPTRAISE_DOUBLE_FAULT:
6726 {
6727 /*
6728 * Determing a vectoring double #PF condition. Used later, when PGM evaluates
6729 * the second #PF as a guest #PF (and not a shadow #PF) and needs to be
6730 * converted into a #DF.
6731 */
6732 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6733 {
6734 Log4Func(("IDT: Pending vectoring double #PF uCR2=%#RX64\n", pVCpu->cpum.GstCtx.cr2));
6735 pSvmTransient->fVectoringDoublePF = true;
6736 Assert(rc == VINF_SUCCESS);
6737 }
6738 else
6739 {
6740 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectConvertDF);
6741 hmR0SvmSetPendingXcptDF(pVCpu);
6742 rc = VINF_HM_DOUBLE_FAULT;
6743 }
6744 break;
6745 }
6746
6747 case IEMXCPTRAISE_TRIPLE_FAULT:
6748 {
6749 rc = VINF_EM_RESET;
6750 break;
6751 }
6752
6753 case IEMXCPTRAISE_CPU_HANG:
6754 {
6755 rc = VERR_EM_GUEST_CPU_HANG;
6756 break;
6757 }
6758
6759 default:
6760 AssertMsgFailedBreakStmt(("Bogus enmRaise value: %d (%#x)\n", enmRaise, enmRaise), rc = VERR_SVM_IPE_2);
6761 }
6762 }
6763 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET || rc == VERR_EM_GUEST_CPU_HANG);
6764 return rc;
6765}
6766
6767
6768/**
6769 * Advances the guest RIP by the number of bytes specified in @a cb.
6770 *
6771 * @param pVCpu The cross context virtual CPU structure.
6772 * @param cb RIP increment value in bytes.
6773 */
6774DECLINLINE(void) hmR0SvmAdvanceRip(PVMCPUCC pVCpu, uint32_t cb)
6775{
6776 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6777 pCtx->rip += cb;
6778
6779 /* Update interrupt shadow. */
6780 if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
6781 && pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
6782 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6783}
6784
6785
6786/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6787/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6788/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6789
6790/** @name \#VMEXIT handlers.
6791 * @{
6792 */
6793
6794/**
6795 * \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6796 * signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6797 */
6798HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6799{
6800 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6801
6802 if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6803 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6804 else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6805 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6806
6807 /*
6808 * AMD-V has no preemption timer and the generic periodic preemption timer has no way to
6809 * signal -before- the timer fires if the current interrupt is our own timer or a some
6810 * other host interrupt. We also cannot examine what interrupt it is until the host
6811 * actually take the interrupt.
6812 *
6813 * Going back to executing guest code here unconditionally causes random scheduling
6814 * problems (observed on an AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6815 */
6816 return VINF_EM_RAW_INTERRUPT;
6817}
6818
6819
6820/**
6821 * \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6822 */
6823HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6824{
6825 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6826
6827 VBOXSTRICTRC rcStrict;
6828 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6829 if (fSupportsNextRipSave)
6830 {
6831 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6832 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6833 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6834 rcStrict = IEMExecDecodedWbinvd(pVCpu, cbInstr);
6835 }
6836 else
6837 {
6838 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6839 rcStrict = IEMExecOne(pVCpu);
6840 }
6841
6842 if (rcStrict == VINF_IEM_RAISED_XCPT)
6843 {
6844 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6845 rcStrict = VINF_SUCCESS;
6846 }
6847 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6848 return rcStrict;
6849}
6850
6851
6852/**
6853 * \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6854 */
6855HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6856{
6857 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6858
6859 VBOXSTRICTRC rcStrict;
6860 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6861 if (fSupportsNextRipSave)
6862 {
6863 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
6864 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6865 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6866 rcStrict = IEMExecDecodedInvd(pVCpu, cbInstr);
6867 }
6868 else
6869 {
6870 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6871 rcStrict = IEMExecOne(pVCpu);
6872 }
6873
6874 if (rcStrict == VINF_IEM_RAISED_XCPT)
6875 {
6876 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6877 rcStrict = VINF_SUCCESS;
6878 }
6879 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6880 return rcStrict;
6881}
6882
6883
6884/**
6885 * \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6886 */
6887HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6888{
6889 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6890
6891 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
6892 VBOXSTRICTRC rcStrict;
6893 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
6894 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
6895 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
6896 if (!pExitRec)
6897 {
6898 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6899 if (fSupportsNextRipSave)
6900 {
6901 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6902 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6903 rcStrict = IEMExecDecodedCpuid(pVCpu, cbInstr);
6904 }
6905 else
6906 {
6907 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6908 rcStrict = IEMExecOne(pVCpu);
6909 }
6910
6911 if (rcStrict == VINF_IEM_RAISED_XCPT)
6912 {
6913 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6914 rcStrict = VINF_SUCCESS;
6915 }
6916 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6917 }
6918 else
6919 {
6920 /*
6921 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
6922 */
6923 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6924
6925 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
6926 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
6927
6928 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
6929
6930 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
6931 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
6932 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
6933 }
6934 return rcStrict;
6935}
6936
6937
6938/**
6939 * \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6940 */
6941HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6942{
6943 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6944
6945 VBOXSTRICTRC rcStrict;
6946 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6947 if (fSupportsNextRipSave)
6948 {
6949 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
6950 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6951 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6952 rcStrict = IEMExecDecodedRdtsc(pVCpu, cbInstr);
6953 }
6954 else
6955 {
6956 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6957 rcStrict = IEMExecOne(pVCpu);
6958 }
6959
6960 if (rcStrict == VINF_SUCCESS)
6961 pSvmTransient->fUpdateTscOffsetting = true;
6962 else if (rcStrict == VINF_IEM_RAISED_XCPT)
6963 {
6964 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6965 rcStrict = VINF_SUCCESS;
6966 }
6967 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6968 return rcStrict;
6969}
6970
6971
6972/**
6973 * \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6974 */
6975HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
6976{
6977 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
6978
6979 VBOXSTRICTRC rcStrict;
6980 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
6981 if (fSupportsNextRipSave)
6982 {
6983 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_TSC_AUX);
6984 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
6985 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
6986 rcStrict = IEMExecDecodedRdtscp(pVCpu, cbInstr);
6987 }
6988 else
6989 {
6990 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
6991 rcStrict = IEMExecOne(pVCpu);
6992 }
6993
6994 if (rcStrict == VINF_SUCCESS)
6995 pSvmTransient->fUpdateTscOffsetting = true;
6996 else if (rcStrict == VINF_IEM_RAISED_XCPT)
6997 {
6998 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
6999 rcStrict = VINF_SUCCESS;
7000 }
7001 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7002 return rcStrict;
7003}
7004
7005
7006/**
7007 * \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
7008 */
7009HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7010{
7011 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7012
7013 VBOXSTRICTRC rcStrict;
7014 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7015 if (fSupportsNextRipSave)
7016 {
7017 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR4);
7018 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7019 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7020 rcStrict = IEMExecDecodedRdpmc(pVCpu, cbInstr);
7021 }
7022 else
7023 {
7024 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7025 rcStrict = IEMExecOne(pVCpu);
7026 }
7027
7028 if (rcStrict == VINF_IEM_RAISED_XCPT)
7029 {
7030 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7031 rcStrict = VINF_SUCCESS;
7032 }
7033 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7034 return rcStrict;
7035}
7036
7037
7038/**
7039 * \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
7040 */
7041HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7042{
7043 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7044 Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
7045
7046 VBOXSTRICTRC rcStrict;
7047 bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7048 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7049 if ( fSupportsDecodeAssists
7050 && fSupportsNextRipSave)
7051 {
7052 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
7053 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7054 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7055 RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
7056 rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
7057 }
7058 else
7059 {
7060 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7061 rcStrict = IEMExecOne(pVCpu);
7062 }
7063
7064 if (rcStrict == VINF_IEM_RAISED_XCPT)
7065 {
7066 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7067 rcStrict = VINF_SUCCESS;
7068 }
7069 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7070 return VBOXSTRICTRC_VAL(rcStrict);
7071}
7072
7073
7074/**
7075 * \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
7076 */
7077HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7078{
7079 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7080
7081 VBOXSTRICTRC rcStrict;
7082 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7083 if (fSupportsNextRipSave)
7084 {
7085 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7086 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7087 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7088 rcStrict = IEMExecDecodedHlt(pVCpu, cbInstr);
7089 }
7090 else
7091 {
7092 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7093 rcStrict = IEMExecOne(pVCpu);
7094 }
7095
7096 if ( rcStrict == VINF_EM_HALT
7097 || rcStrict == VINF_SUCCESS)
7098 rcStrict = EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx) ? VINF_SUCCESS : VINF_EM_HALT;
7099 else if (rcStrict == VINF_IEM_RAISED_XCPT)
7100 {
7101 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7102 rcStrict = VINF_SUCCESS;
7103 }
7104 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7105 if (rcStrict != VINF_SUCCESS)
7106 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
7107 return VBOXSTRICTRC_VAL(rcStrict);;
7108}
7109
7110
7111/**
7112 * \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
7113 */
7114HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7115{
7116 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7117
7118 /*
7119 * If the instruction length is supplied by the CPU is 3 bytes, we can be certain that no
7120 * segment override prefix is present (and thus use the default segment DS). Otherwise, a
7121 * segment override prefix or other prefixes might be used, in which case we fallback to
7122 * IEMExecOne() to figure out.
7123 */
7124 VBOXSTRICTRC rcStrict;
7125 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7126 uint8_t const cbInstr = hmR0SvmSupportsNextRipSave(pVCpu) ? pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip : 0;
7127 if (cbInstr)
7128 {
7129 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
7130 rcStrict = IEMExecDecodedMonitor(pVCpu, cbInstr);
7131 }
7132 else
7133 {
7134 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7135 rcStrict = IEMExecOne(pVCpu);
7136 }
7137
7138 if (rcStrict == VINF_IEM_RAISED_XCPT)
7139 {
7140 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7141 rcStrict = VINF_SUCCESS;
7142 }
7143 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7144 return rcStrict;
7145}
7146
7147
7148/**
7149 * \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
7150 */
7151HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7152{
7153 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7154
7155 VBOXSTRICTRC rcStrict;
7156 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7157 if (fSupportsNextRipSave)
7158 {
7159 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
7160 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7161 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7162 rcStrict = IEMExecDecodedMwait(pVCpu, cbInstr);
7163 }
7164 else
7165 {
7166 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7167 rcStrict = IEMExecOne(pVCpu);
7168 }
7169
7170 if ( rcStrict == VINF_EM_HALT
7171 && EMMonitorWaitShouldContinue(pVCpu, &pVCpu->cpum.GstCtx))
7172 rcStrict = VINF_SUCCESS;
7173 else if (rcStrict == VINF_IEM_RAISED_XCPT)
7174 {
7175 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7176 rcStrict = VINF_SUCCESS;
7177 }
7178 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7179 return rcStrict;
7180}
7181
7182
7183/**
7184 * \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
7185 * \#VMEXIT.
7186 */
7187HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7188{
7189 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7190 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7191 return VINF_EM_RESET;
7192}
7193
7194
7195/**
7196 * \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
7197 */
7198HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7199{
7200 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7201 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7202 AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64 uExitInfo1=%#RX64 uExitInfo2=%#RX64\n", pSvmTransient->u64ExitCode,
7203 pVmcb->ctrl.u64ExitInfo1, pVmcb->ctrl.u64ExitInfo2));
7204 RT_NOREF(pVmcb);
7205 pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7206 return VERR_SVM_UNEXPECTED_EXIT;
7207}
7208
7209
7210/**
7211 * \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
7212 */
7213HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7214{
7215 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7216
7217 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7218 Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7219#ifdef VBOX_WITH_STATISTICS
7220 switch (pSvmTransient->u64ExitCode)
7221 {
7222 case SVM_EXIT_READ_CR0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
7223 case SVM_EXIT_READ_CR2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
7224 case SVM_EXIT_READ_CR3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
7225 case SVM_EXIT_READ_CR4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
7226 case SVM_EXIT_READ_CR8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
7227 }
7228#endif
7229
7230 bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7231 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7232 if ( fSupportsDecodeAssists
7233 && fSupportsNextRipSave)
7234 {
7235 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7236 bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7237 if (fMovCRx)
7238 {
7239 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_CR_MASK
7240 | CPUMCTX_EXTRN_APIC_TPR);
7241 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7242 uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
7243 uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7244 VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
7245 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7246 return VBOXSTRICTRC_VAL(rcStrict);
7247 }
7248 /* else: SMSW instruction, fall back below to IEM for this. */
7249 }
7250
7251 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7252 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7253 AssertMsg( rcStrict == VINF_SUCCESS
7254 || rcStrict == VINF_PGM_SYNC_CR3
7255 || rcStrict == VINF_IEM_RAISED_XCPT,
7256 ("hmR0SvmExitReadCRx: IEMExecOne failed rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7257 Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
7258 if (rcStrict == VINF_IEM_RAISED_XCPT)
7259 {
7260 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7261 rcStrict = VINF_SUCCESS;
7262 }
7263 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7264 return rcStrict;
7265}
7266
7267
7268/**
7269 * \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
7270 */
7271HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7272{
7273 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7274
7275 uint64_t const uExitCode = pSvmTransient->u64ExitCode;
7276 uint8_t const iCrReg = uExitCode == SVM_EXIT_CR0_SEL_WRITE ? 0 : (pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0);
7277 Assert(iCrReg <= 15);
7278
7279 VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
7280 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7281 bool fDecodedInstr = false;
7282 bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu);
7283 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7284 if ( fSupportsDecodeAssists
7285 && fSupportsNextRipSave)
7286 {
7287 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7288 bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
7289 if (fMovCRx)
7290 {
7291 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4
7292 | CPUMCTX_EXTRN_APIC_TPR);
7293 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
7294 uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
7295 Log4Func(("Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
7296 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
7297 fDecodedInstr = true;
7298 }
7299 /* else: LMSW or CLTS instruction, fall back below to IEM for this. */
7300 }
7301
7302 if (!fDecodedInstr)
7303 {
7304 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7305 Log4Func(("iCrReg=%#x\n", iCrReg));
7306 rcStrict = IEMExecOne(pVCpu);
7307 if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
7308 || rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
7309 rcStrict = VERR_EM_INTERPRETER;
7310 }
7311
7312 if (rcStrict == VINF_SUCCESS)
7313 {
7314 switch (iCrReg)
7315 {
7316 case 0:
7317 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
7318 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
7319 break;
7320
7321 case 2:
7322 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR2);
7323 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
7324 break;
7325
7326 case 3:
7327 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
7328 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
7329 break;
7330
7331 case 4:
7332 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
7333 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
7334 break;
7335
7336 case 8:
7337 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7338 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
7339 break;
7340
7341 default:
7342 {
7343 AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
7344 pSvmTransient->u64ExitCode, iCrReg));
7345 break;
7346 }
7347 }
7348 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7349 }
7350 else if (rcStrict == VINF_IEM_RAISED_XCPT)
7351 {
7352 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7353 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7354 rcStrict = VINF_SUCCESS;
7355 }
7356 else
7357 Assert(rcStrict == VERR_EM_INTERPRETER || rcStrict == VINF_PGM_SYNC_CR3);
7358 return rcStrict;
7359}
7360
7361
7362/**
7363 * \#VMEXIT helper for read MSRs, see hmR0SvmExitMsr.
7364 *
7365 * @returns Strict VBox status code.
7366 * @param pVCpu The cross context virtual CPU structure.
7367 * @param pVmcb Pointer to the VM control block.
7368 */
7369static VBOXSTRICTRC hmR0SvmExitReadMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb)
7370{
7371 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7372 Log4Func(("idMsr=%#RX32\n", pVCpu->cpum.GstCtx.ecx));
7373
7374 VBOXSTRICTRC rcStrict;
7375 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7376 if (fSupportsNextRipSave)
7377 {
7378 /** @todo Optimize this: Only retrieve the MSR bits we need here. CPUMAllMsrs.cpp
7379 * can ask for what it needs instead of using CPUMCTX_EXTRN_ALL_MSRS. */
7380 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
7381 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7382 rcStrict = IEMExecDecodedRdmsr(pVCpu, cbInstr);
7383 }
7384 else
7385 {
7386 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_ALL_MSRS);
7387 rcStrict = IEMExecOne(pVCpu);
7388 }
7389
7390 AssertMsg( rcStrict == VINF_SUCCESS
7391 || rcStrict == VINF_IEM_RAISED_XCPT
7392 || rcStrict == VINF_CPUM_R3_MSR_READ,
7393 ("hmR0SvmExitReadMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7394
7395 if (rcStrict == VINF_IEM_RAISED_XCPT)
7396 {
7397 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7398 rcStrict = VINF_SUCCESS;
7399 }
7400 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7401 return rcStrict;
7402}
7403
7404
7405/**
7406 * \#VMEXIT helper for write MSRs, see hmR0SvmExitMsr.
7407 *
7408 * @returns Strict VBox status code.
7409 * @param pVCpu The cross context virtual CPU structure.
7410 * @param pVmcb Pointer to the VM control block.
7411 * @param pSvmTransient Pointer to the SVM-transient structure.
7412 */
7413static VBOXSTRICTRC hmR0SvmExitWriteMsr(PVMCPUCC pVCpu, PSVMVMCB pVmcb, PSVMTRANSIENT pSvmTransient)
7414{
7415 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7416 uint32_t const idMsr = pCtx->ecx;
7417 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7418 Log4Func(("idMsr=%#RX32\n", idMsr));
7419
7420 /*
7421 * Handle TPR patching MSR writes.
7422 * We utilitize the LSTAR MSR for patching.
7423 */
7424 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7425 if ( idMsr == MSR_K8_LSTAR
7426 && pVCpu->CTX_SUFF(pVM)->hm.s.fTprPatchingActive)
7427 {
7428 unsigned cbInstr;
7429 if (fSupportsNextRipSave)
7430 cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7431 else
7432 {
7433 PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
7434 int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
7435 if ( rc == VINF_SUCCESS
7436 && pDis->pCurInstr->uOpcode == OP_WRMSR)
7437 Assert(cbInstr > 0);
7438 else
7439 cbInstr = 0;
7440 }
7441
7442 /* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
7443 if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
7444 {
7445 int rc = APICSetTpr(pVCpu, pCtx->eax & 0xff);
7446 AssertRCReturn(rc, rc);
7447 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7448 }
7449
7450 int rc = VINF_SUCCESS;
7451 hmR0SvmAdvanceRip(pVCpu, cbInstr);
7452 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7453 return rc;
7454 }
7455
7456 /*
7457 * Handle regular MSR writes.
7458 */
7459 VBOXSTRICTRC rcStrict;
7460 if (fSupportsNextRipSave)
7461 {
7462 /** @todo Optimize this: We don't need to get much of the MSR state here
7463 * since we're only updating. CPUMAllMsrs.cpp can ask for what it needs and
7464 * clear the applicable extern flags. */
7465 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
7466 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
7467 rcStrict = IEMExecDecodedWrmsr(pVCpu, cbInstr);
7468 }
7469 else
7470 {
7471 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_ALL_MSRS);
7472 rcStrict = IEMExecOne(pVCpu);
7473 }
7474
7475 AssertMsg( rcStrict == VINF_SUCCESS
7476 || rcStrict == VINF_IEM_RAISED_XCPT
7477 || rcStrict == VINF_CPUM_R3_MSR_WRITE,
7478 ("hmR0SvmExitWriteMsr: Unexpected status %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7479
7480 if (rcStrict == VINF_SUCCESS)
7481 {
7482 /* If this is an X2APIC WRMSR access, update the APIC TPR state. */
7483 if ( idMsr >= MSR_IA32_X2APIC_START
7484 && idMsr <= MSR_IA32_X2APIC_END)
7485 {
7486 /*
7487 * We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest().
7488 * When full APIC register virtualization is implemented we'll have to make sure
7489 * APIC state is saved from the VMCB before IEM changes it.
7490 */
7491 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7492 }
7493 else
7494 {
7495 switch (idMsr)
7496 {
7497 case MSR_IA32_TSC: pSvmTransient->fUpdateTscOffsetting = true; break;
7498 case MSR_K6_EFER: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR); break;
7499 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
7500 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
7501 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
7502 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
7503 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
7504 }
7505 }
7506 }
7507 else if (rcStrict == VINF_IEM_RAISED_XCPT)
7508 {
7509 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7510 rcStrict = VINF_SUCCESS;
7511 }
7512 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7513 return rcStrict;
7514}
7515
7516
7517/**
7518 * \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
7519 * \#VMEXIT.
7520 */
7521HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7522{
7523 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7524
7525 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7526 if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ)
7527 return hmR0SvmExitReadMsr(pVCpu, pVmcb);
7528
7529 Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE);
7530 return hmR0SvmExitWriteMsr(pVCpu, pVmcb, pSvmTransient);
7531}
7532
7533
7534/**
7535 * \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
7536 */
7537HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7538{
7539 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7540 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7541
7542 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7543
7544 /** @todo Stepping with nested-guest. */
7545 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7546 if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
7547 {
7548 /* We should -not- get this #VMEXIT if the guest's debug registers were active. */
7549 if (pSvmTransient->fWasGuestDebugStateActive)
7550 {
7551 AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
7552 pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
7553 return VERR_SVM_UNEXPECTED_EXIT;
7554 }
7555
7556 /*
7557 * Lazy DR0-3 loading.
7558 */
7559 if (!pSvmTransient->fWasHyperDebugStateActive)
7560 {
7561 Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
7562 Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
7563
7564 /* Don't intercept DRx read and writes. */
7565 PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
7566 pVmcb->ctrl.u16InterceptRdDRx = 0;
7567 pVmcb->ctrl.u16InterceptWrDRx = 0;
7568 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
7569
7570 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7571 VMMRZCallRing3Disable(pVCpu);
7572 HM_DISABLE_PREEMPT(pVCpu);
7573
7574 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7575 CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
7576 Assert(CPUMIsGuestDebugStateActive(pVCpu));
7577
7578 HM_RESTORE_PREEMPT();
7579 VMMRZCallRing3Enable(pVCpu);
7580
7581 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7582 return VINF_SUCCESS;
7583 }
7584 }
7585
7586 /*
7587 * Interpret the read/writing of DRx.
7588 */
7589 /** @todo Decode assist. */
7590 VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
7591 Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
7592 if (RT_LIKELY(rc == VINF_SUCCESS))
7593 {
7594 /* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
7595 /** @todo CPUM should set this flag! */
7596 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR_MASK);
7597 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7598 }
7599 else
7600 Assert(rc == VERR_EM_INTERPRETER);
7601 return rc;
7602}
7603
7604
7605/**
7606 * \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
7607 */
7608HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7609{
7610 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7611 /* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
7612 VBOXSTRICTRC rc = hmR0SvmExitReadDRx(pVCpu, pSvmTransient);
7613 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7614 STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
7615 return rc;
7616}
7617
7618
7619/**
7620 * \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
7621 */
7622HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7623{
7624 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7625 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
7626
7627 /** @todo decode assists... */
7628 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
7629 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
7630 {
7631 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7632 bool const fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
7633 Log4Func(("New XCR0=%#RX64 fLoadSaveGuestXcr0=%RTbool (cr4=%#RX64)\n", pCtx->aXcr[0], fLoadSaveGuestXcr0, pCtx->cr4));
7634 if (fLoadSaveGuestXcr0 != pVCpu->hmr0.s.fLoadSaveGuestXcr0)
7635 {
7636 pVCpu->hmr0.s.fLoadSaveGuestXcr0 = fLoadSaveGuestXcr0;
7637 hmR0SvmUpdateVmRunFunction(pVCpu);
7638 }
7639 }
7640 else if (rcStrict == VINF_IEM_RAISED_XCPT)
7641 {
7642 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7643 rcStrict = VINF_SUCCESS;
7644 }
7645 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7646 return rcStrict;
7647}
7648
7649
7650/**
7651 * \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
7652 */
7653HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7654{
7655 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7656 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK);
7657
7658 /* I/O operation lookup arrays. */
7659 static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
7660 static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
7661 the result (in AL/AX/EAX). */
7662 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7663 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7664 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7665
7666 Log4Func(("CS:RIP=%04x:%RX64\n", pCtx->cs.Sel, pCtx->rip));
7667
7668 /* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
7669 SVMIOIOEXITINFO IoExitInfo;
7670 IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
7671 uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
7672 uint32_t cbValue = s_aIOSize[uIOWidth];
7673 uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7674
7675 if (RT_UNLIKELY(!cbValue))
7676 {
7677 AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
7678 return VERR_EM_INTERPRETER;
7679 }
7680
7681 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
7682 VBOXSTRICTRC rcStrict;
7683 PCEMEXITREC pExitRec = NULL;
7684 if ( !pVCpu->hm.s.fSingleInstruction
7685 && !pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
7686 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7687 !IoExitInfo.n.u1Str
7688 ? IoExitInfo.n.u1Type == SVM_IOIO_READ
7689 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
7690 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
7691 : IoExitInfo.n.u1Type == SVM_IOIO_READ
7692 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
7693 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
7694 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7695 if (!pExitRec)
7696 {
7697 bool fUpdateRipAlready = false;
7698 if (IoExitInfo.n.u1Str)
7699 {
7700 /* INS/OUTS - I/O String instruction. */
7701 /** @todo Huh? why can't we use the segment prefix information given by AMD-V
7702 * in EXITINFO1? Investigate once this thing is up and running. */
7703 Log4Func(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
7704 IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
7705 AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
7706 static IEMMODE const s_aenmAddrMode[8] =
7707 {
7708 (IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
7709 };
7710 IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
7711 if (enmAddrMode != (IEMMODE)-1)
7712 {
7713 uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7714 if (cbInstr <= 15 && cbInstr >= 1)
7715 {
7716 Assert(cbInstr >= 1U + IoExitInfo.n.u1Rep);
7717 if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7718 {
7719 /* Don't know exactly how to detect whether u3Seg is valid, currently
7720 only enabling it for Bulldozer and later with NRIP. OS/2 broke on
7721 2384 Opterons when only checking NRIP. */
7722 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
7723 if ( fSupportsNextRipSave
7724 && pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
7725 {
7726 AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_DS || cbInstr > 1U + IoExitInfo.n.u1Rep,
7727 ("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3Seg, cbInstr, IoExitInfo.n.u1Rep));
7728 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7729 IoExitInfo.n.u3Seg, true /*fIoChecked*/);
7730 }
7731 else if (cbInstr == 1U + IoExitInfo.n.u1Rep)
7732 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7733 X86_SREG_DS, true /*fIoChecked*/);
7734 else
7735 rcStrict = IEMExecOne(pVCpu);
7736 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7737 }
7738 else
7739 {
7740 AssertMsg(IoExitInfo.n.u3Seg == X86_SREG_ES /*=0*/, ("%#x\n", IoExitInfo.n.u3Seg));
7741 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1Rep, (uint8_t)cbInstr,
7742 true /*fIoChecked*/);
7743 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7744 }
7745 }
7746 else
7747 {
7748 AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
7749 rcStrict = IEMExecOne(pVCpu);
7750 }
7751 }
7752 else
7753 {
7754 AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
7755 rcStrict = IEMExecOne(pVCpu);
7756 }
7757 fUpdateRipAlready = true;
7758 if (rcStrict == VINF_IEM_RAISED_XCPT)
7759 {
7760 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
7761 rcStrict = VINF_SUCCESS;
7762 }
7763 }
7764 else
7765 {
7766 /* IN/OUT - I/O instruction. */
7767 Assert(!IoExitInfo.n.u1Rep);
7768
7769 uint8_t const cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
7770 if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
7771 {
7772 rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
7773 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7774 && !pCtx->eflags.Bits.u1TF)
7775 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue, pCtx->eax & uAndVal);
7776 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7777 }
7778 else
7779 {
7780 uint32_t u32Val = 0;
7781 rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
7782 if (IOM_SUCCESS(rcStrict))
7783 {
7784 /* Save result of I/O IN instr. in AL/AX/EAX. */
7785 /** @todo r=bird: 32-bit op size should clear high bits of rax! */
7786 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
7787 }
7788 else if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7789 && !pCtx->eflags.Bits.u1TF)
7790 rcStrict = EMRZSetPendingIoPortRead(pVCpu, IoExitInfo.n.u16Port, cbInstr, cbValue);
7791
7792 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7793 }
7794 }
7795
7796 if (IOM_SUCCESS(rcStrict))
7797 {
7798 /* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
7799 if (!fUpdateRipAlready)
7800 pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
7801
7802 /*
7803 * If any I/O breakpoints are armed, we need to check if one triggered
7804 * and take appropriate action.
7805 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
7806 */
7807 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
7808 * execution engines about whether hyper BPs and such are pending. */
7809 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_DR7);
7810 uint32_t const uDr7 = pCtx->dr[7];
7811 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
7812 && X86_DR7_ANY_RW_IO(uDr7)
7813 && (pCtx->cr4 & X86_CR4_DE))
7814 || DBGFBpIsHwIoArmed(pVM)))
7815 {
7816 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7817 VMMRZCallRing3Disable(pVCpu);
7818 HM_DISABLE_PREEMPT(pVCpu);
7819
7820 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7821 CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /*fDr6*/);
7822
7823 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, &pVCpu->cpum.GstCtx, IoExitInfo.n.u16Port, cbValue);
7824 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
7825 {
7826 /* Raise #DB. */
7827 pVmcb->guest.u64DR6 = pCtx->dr[6];
7828 pVmcb->guest.u64DR7 = pCtx->dr[7];
7829 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7830 hmR0SvmSetPendingXcptDB(pVCpu);
7831 }
7832 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
7833 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
7834 else if ( rcStrict2 != VINF_SUCCESS
7835 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
7836 rcStrict = rcStrict2;
7837 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
7838
7839 HM_RESTORE_PREEMPT();
7840 VMMRZCallRing3Enable(pVCpu);
7841 }
7842
7843 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
7844 }
7845#ifdef VBOX_STRICT
7846 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
7847 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
7848 Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
7849 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
7850 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
7851 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
7852 Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
7853 else
7854 {
7855 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
7856 * statuses, that the VMM device and some others may return. See
7857 * IOM_SUCCESS() for guidance. */
7858 AssertMsg( RT_FAILURE(rcStrict)
7859 || rcStrict == VINF_SUCCESS
7860 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
7861 || rcStrict == VINF_EM_DBG_BREAKPOINT
7862 || rcStrict == VINF_EM_RAW_GUEST_TRAP
7863 || rcStrict == VINF_EM_DBG_STEPPED
7864 || rcStrict == VINF_EM_RAW_TO_R3
7865 || rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7866 }
7867#endif
7868 }
7869 else
7870 {
7871 /*
7872 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
7873 */
7874 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7875 STAM_COUNTER_INC(!IoExitInfo.n.u1Str
7876 ? IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
7877 : IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
7878 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
7879 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, IoExitInfo.n.u1Rep ? "REP " : "",
7880 IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? "OUT" : "IN", IoExitInfo.n.u1Str ? "S" : "", IoExitInfo.n.u16Port, uIOWidth));
7881
7882 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
7883 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7884
7885 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
7886 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
7887 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
7888 }
7889 return rcStrict;
7890}
7891
7892
7893/**
7894 * \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
7895 */
7896HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
7897{
7898 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
7899 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7900 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
7901
7902 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
7903 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7904 Assert(pVM->hmr0.s.fNestedPaging);
7905
7906 /* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
7907 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
7908 RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
7909 uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1; /* Note! High bits in EXITINFO1 may contain additional info and are
7910 thus intentionally not copied into u32ErrCode. */
7911
7912 Log4Func(("#NPF at CS:RIP=%04x:%RX64 GCPhysFaultAddr=%RGp ErrCode=%#x cbInstrFetched=%u %.15Rhxs\n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr,
7913 u32ErrCode, pVmcb->ctrl.cbInstrFetched, pVmcb->ctrl.abInstr));
7914
7915 /*
7916 * TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions.
7917 */
7918 if ( pVM->hm.s.fTprPatchingAllowed
7919 && (GCPhysFaultAddr & GUEST_PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
7920 && ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
7921 || (u32ErrCode & (X86_TRAP_PF_P | X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P | X86_TRAP_PF_RSVD)) /* MMIO page. */
7922 && !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
7923 && !CPUMIsGuestInLongModeEx(pCtx)
7924 && !CPUMGetGuestCPL(pVCpu)
7925 && pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7926 {
7927 RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7928 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7929
7930 if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
7931 {
7932 /* Only attempt to patch the instruction once. */
7933 PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7934 if (!pPatch)
7935 return VINF_EM_HM_PATCH_TPR_INSTR;
7936 }
7937 }
7938
7939 /*
7940 * Determine the nested paging mode.
7941 */
7942/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
7943 PGMMODE const enmNestedPagingMode = PGMGetHostMode(pVM);
7944
7945 /*
7946 * MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
7947 */
7948 Assert((u32ErrCode & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
7949 if ((u32ErrCode & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD | X86_TRAP_PF_P))
7950 {
7951 /*
7952 * If event delivery causes an MMIO #NPF, go back to instruction emulation as otherwise
7953 * injecting the original pending event would most likely cause the same MMIO #NPF.
7954 */
7955 if (pVCpu->hm.s.Event.fPending)
7956 {
7957 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
7958 return VINF_EM_RAW_INJECT_TRPM_EVENT;
7959 }
7960
7961 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
7962 VBOXSTRICTRC rcStrict;
7963 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
7964 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
7965 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
7966 if (!pExitRec)
7967 {
7968
7969 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
7970 u32ErrCode);
7971
7972 /*
7973 * If we succeed, resume guest execution.
7974 *
7975 * If we fail in interpreting the instruction because we couldn't get the guest
7976 * physical address of the page containing the instruction via the guest's page
7977 * tables (we would invalidate the guest page in the host TLB), resume execution
7978 * which would cause a guest page fault to let the guest handle this weird case.
7979 *
7980 * See @bugref{6043}.
7981 */
7982 if ( rcStrict == VINF_SUCCESS
7983 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
7984 || rcStrict == VERR_PAGE_NOT_PRESENT)
7985 {
7986 /* Successfully handled MMIO operation. */
7987 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
7988 rcStrict = VINF_SUCCESS;
7989 }
7990 }
7991 else
7992 {
7993 /*
7994 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
7995 */
7996 Assert(pCtx == &pVCpu->cpum.GstCtx);
7997 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
7998 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
7999 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhysFaultAddr));
8000
8001 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
8002 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8003
8004 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
8005 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
8006 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8007 }
8008 return rcStrict;
8009 }
8010
8011 /*
8012 * Nested page-fault.
8013 */
8014 TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
8015 int rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
8016 TRPMResetTrap(pVCpu);
8017
8018 Log4Func(("#NPF: PGMR0Trap0eHandlerNestedPaging returns %Rrc CS:RIP=%04x:%RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
8019
8020 /*
8021 * Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
8022 */
8023 if ( rc == VINF_SUCCESS
8024 || rc == VERR_PAGE_TABLE_NOT_PRESENT
8025 || rc == VERR_PAGE_NOT_PRESENT)
8026 {
8027 /* We've successfully synced our shadow page tables. */
8028 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8029 rc = VINF_SUCCESS;
8030 }
8031
8032 /*
8033 * If delivering an event causes an #NPF (and not MMIO), we shall resolve the fault and
8034 * re-inject the original event.
8035 */
8036 if (pVCpu->hm.s.Event.fPending)
8037 {
8038 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectReflectNPF);
8039
8040 /*
8041 * If the #NPF handler requested emulation of the instruction, ignore it.
8042 * We need to re-inject the original event so as to not lose it.
8043 * Reproducible when booting ReactOS 0.4.12 with BTRFS (installed using BootCD,
8044 * LiveCD is broken for other reasons).
8045 */
8046 if (rc == VINF_EM_RAW_EMULATE_INSTR)
8047 rc = VINF_EM_RAW_INJECT_TRPM_EVENT;
8048 }
8049
8050 return rc;
8051}
8052
8053
8054/**
8055 * \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
8056 * \#VMEXIT.
8057 */
8058HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8059{
8060 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8061 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8062
8063 /* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
8064 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8065 hmR0SvmClearIntWindowExiting(pVCpu, pVmcb);
8066
8067 /* Deliver the pending interrupt via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
8068 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8069 return VINF_SUCCESS;
8070}
8071
8072
8073/**
8074 * \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
8075 * \#VMEXIT.
8076 */
8077HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8078{
8079 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8080 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8081
8082#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
8083 Assert(!pVCpu->CTX_SUFF(pVM)->hmr0.s.fNestedPaging);
8084#endif
8085
8086 /* Check if this task-switch occurred while delivering an event through the guest IDT. */
8087 if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
8088 {
8089 /*
8090 * AMD-V provides us with the exception which caused the TS; we collect
8091 * the information in the call to hmR0SvmCheckExitDueToEventDelivery().
8092 */
8093 Log4Func(("TS occurred during event delivery\n"));
8094 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8095 return VINF_EM_RAW_INJECT_TRPM_EVENT;
8096 }
8097
8098 /** @todo Emulate task switch someday, currently just going back to ring-3 for
8099 * emulation. */
8100 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8101 return VERR_EM_INTERPRETER;
8102}
8103
8104
8105/**
8106 * \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8107 */
8108HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8109{
8110 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8111 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8112
8113 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8114 if (pVM->hm.s.fTprPatchingAllowed)
8115 {
8116 int rc = hmEmulateSvmMovTpr(pVM, pVCpu);
8117 if (rc != VERR_NOT_FOUND)
8118 {
8119 Log4Func(("hmEmulateSvmMovTpr returns %Rrc\n", rc));
8120 return rc;
8121 }
8122 }
8123
8124 if (EMAreHypercallInstructionsEnabled(pVCpu))
8125 {
8126 unsigned cbInstr;
8127 if (hmR0SvmSupportsNextRipSave(pVCpu))
8128 {
8129 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8130 cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8131 }
8132 else
8133 {
8134 PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8135 int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
8136 if ( rc == VINF_SUCCESS
8137 && pDis->pCurInstr->uOpcode == OP_VMMCALL)
8138 Assert(cbInstr > 0);
8139 else
8140 cbInstr = 0;
8141 }
8142
8143 VBOXSTRICTRC rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
8144 if (RT_SUCCESS(rcStrict))
8145 {
8146 /* Only update the RIP if we're continuing guest execution and not in the case
8147 of say VINF_GIM_R3_HYPERCALL. */
8148 if (rcStrict == VINF_SUCCESS)
8149 hmR0SvmAdvanceRip(pVCpu, cbInstr);
8150
8151 return VBOXSTRICTRC_VAL(rcStrict);
8152 }
8153 else
8154 Log4Func(("GIMHypercall returns %Rrc -> #UD\n", VBOXSTRICTRC_VAL(rcStrict)));
8155 }
8156
8157 hmR0SvmSetPendingXcptUD(pVCpu);
8158 return VINF_SUCCESS;
8159}
8160
8161
8162/**
8163 * \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
8164 */
8165HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8166{
8167 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8168
8169 unsigned cbInstr;
8170 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8171 if (fSupportsNextRipSave)
8172 {
8173 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8174 cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8175 }
8176 else
8177 {
8178 PDISCPUSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8179 int rc = EMInterpretDisasCurrent(pVCpu->CTX_SUFF(pVM), pVCpu, pDis, &cbInstr);
8180 if ( rc == VINF_SUCCESS
8181 && pDis->pCurInstr->uOpcode == OP_PAUSE)
8182 Assert(cbInstr > 0);
8183 else
8184 cbInstr = 0;
8185 }
8186
8187 /** @todo The guest has likely hit a contended spinlock. We might want to
8188 * poke a schedule different guest VCPU. */
8189 hmR0SvmAdvanceRip(pVCpu, cbInstr);
8190 return VINF_EM_RAW_INTERRUPT;
8191}
8192
8193
8194/**
8195 * \#VMEXIT handler for FERR intercept (SVM_EXIT_FERR_FREEZE). Conditional
8196 * \#VMEXIT.
8197 */
8198HMSVM_EXIT_DECL hmR0SvmExitFerrFreeze(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8199{
8200 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8201 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CR0);
8202 Assert(!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE));
8203
8204 Log4Func(("Raising IRQ 13 in response to #FERR\n"));
8205 return PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq */, 1 /* u8Level */, 0 /* uTagSrc */);
8206}
8207
8208
8209/**
8210 * \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
8211 */
8212HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8213{
8214 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8215
8216 /* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now (almost) ready. */
8217 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8218 hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_IRET);
8219
8220 /* Emulate the IRET. We have to execute the IRET before an NMI, but must potentially
8221 * deliver a pending NMI right after. If the IRET faults, an NMI can come before the
8222 * handler executes. Yes, x86 is ugly.
8223 */
8224 return VINF_EM_RAW_EMULATE_INSTR;
8225}
8226
8227
8228/**
8229 * \#VMEXIT handler for page-fault exceptions (SVM_EXIT_XCPT_14).
8230 * Conditional \#VMEXIT.
8231 */
8232HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8233{
8234 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8235 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8236 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8237
8238 /* See AMD spec. 15.12.15 "#PF (Page Fault)". */
8239 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8240 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8241 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8242 uint32_t uErrCode = pVmcb->ctrl.u64ExitInfo1;
8243 uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
8244
8245#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) || defined(HMSVM_ALWAYS_TRAP_PF)
8246 if (pVM->hmr0.s.fNestedPaging)
8247 {
8248 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8249 if ( !pSvmTransient->fVectoringDoublePF
8250 || CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8251 {
8252 /* A genuine guest #PF, reflect it to the guest. */
8253 hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8254 Log4Func(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RX64 ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
8255 uFaultAddress, uErrCode));
8256 }
8257 else
8258 {
8259 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8260 hmR0SvmSetPendingXcptDF(pVCpu);
8261 Log4Func(("Pending #DF due to vectoring #PF. NP\n"));
8262 }
8263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8264 return VINF_SUCCESS;
8265 }
8266#endif
8267
8268 Assert(!pVM->hmr0.s.fNestedPaging);
8269
8270 /*
8271 * TPR patching shortcut for APIC TPR reads and writes; only applicable to 32-bit guests.
8272 */
8273 if ( pVM->hm.s.fTprPatchingAllowed
8274 && (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
8275 && !(uErrCode & X86_TRAP_PF_P) /* Not present. */
8276 && !CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8277 && !CPUMIsGuestInLongModeEx(pCtx)
8278 && !CPUMGetGuestCPL(pVCpu)
8279 && pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8280 {
8281 RTGCPHYS GCPhysApicBase;
8282 GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
8283 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8284
8285 /* Check if the page at the fault-address is the APIC base. */
8286 PGMPTWALK Walk;
8287 int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, &Walk);
8288 if ( rc2 == VINF_SUCCESS
8289 && Walk.GCPhys == GCPhysApicBase)
8290 {
8291 /* Only attempt to patch the instruction once. */
8292 PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
8293 if (!pPatch)
8294 return VINF_EM_HM_PATCH_TPR_INSTR;
8295 }
8296 }
8297
8298 Log4Func(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
8299 pCtx->rip, uErrCode, pCtx->cr3));
8300
8301 /*
8302 * If it's a vectoring #PF, emulate injecting the original event injection as
8303 * PGMTrap0eHandler() is incapable of differentiating between instruction emulation and
8304 * event injection that caused a #PF. See @bugref{6607}.
8305 */
8306 if (pSvmTransient->fVectoringPF)
8307 {
8308 Assert(pVCpu->hm.s.Event.fPending);
8309 return VINF_EM_RAW_INJECT_TRPM_EVENT;
8310 }
8311
8312 TRPMAssertXcptPF(pVCpu, uFaultAddress, uErrCode);
8313 int rc = PGMTrap0eHandler(pVCpu, uErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
8314
8315 Log4Func(("#PF: rc=%Rrc\n", rc));
8316
8317 if (rc == VINF_SUCCESS)
8318 {
8319 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8320 TRPMResetTrap(pVCpu);
8321 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8322 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8323 return rc;
8324 }
8325
8326 if (rc == VINF_EM_RAW_GUEST_TRAP)
8327 {
8328 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
8329
8330 /*
8331 * If a nested-guest delivers a #PF and that causes a #PF which is -not- a shadow #PF,
8332 * we should simply forward the #PF to the guest and is up to the nested-hypervisor to
8333 * determine whether it is a nested-shadow #PF or a #DF, see @bugref{7243#c121}.
8334 */
8335 if ( !pSvmTransient->fVectoringDoublePF
8336 || CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
8337 {
8338 /* It's a guest (or nested-guest) page fault and needs to be reflected. */
8339 uErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
8340 TRPMResetTrap(pVCpu);
8341
8342#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8343 /* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
8344 if ( CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
8345 && CPUMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
8346 return IEMExecSvmVmexit(pVCpu, SVM_EXIT_XCPT_PF, uErrCode, uFaultAddress);
8347#endif
8348
8349 hmR0SvmSetPendingXcptPF(pVCpu, uErrCode, uFaultAddress);
8350 }
8351 else
8352 {
8353 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8354 TRPMResetTrap(pVCpu);
8355 hmR0SvmSetPendingXcptDF(pVCpu);
8356 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
8357 }
8358
8359 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8360 return VINF_SUCCESS;
8361 }
8362
8363 TRPMResetTrap(pVCpu);
8364 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8365 return rc;
8366}
8367
8368
8369/**
8370 * \#VMEXIT handler for undefined opcode (SVM_EXIT_XCPT_6).
8371 * Conditional \#VMEXIT.
8372 */
8373HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8374{
8375 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8376 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(&pVCpu->cpum.GstCtx);
8377 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8378
8379 /* Paranoia; Ensure we cannot be called as a result of event delivery. */
8380 PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8381 Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8382
8383 /** @todo if we accumulate more optional stuff here, we ought to combine the
8384 * reading of opcode bytes to avoid doing more than once. */
8385
8386 VBOXSTRICTRC rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8387 if (pVCpu->hm.s.fGIMTrapXcptUD)
8388 {
8389 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8390 uint8_t cbInstr = 0;
8391 rcStrict = GIMXcptUD(pVCpu, &pVCpu->cpum.GstCtx, NULL /* pDis */, &cbInstr);
8392 if (rcStrict == VINF_SUCCESS)
8393 {
8394 /* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
8395 hmR0SvmAdvanceRip(pVCpu, cbInstr);
8396 rcStrict = VINF_SUCCESS;
8397 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8398 }
8399 else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
8400 rcStrict = VINF_SUCCESS;
8401 else if (rcStrict == VINF_GIM_R3_HYPERCALL)
8402 rcStrict = VINF_GIM_R3_HYPERCALL;
8403 else
8404 {
8405 Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
8406 rcStrict = VERR_SVM_UNEXPECTED_XCPT_EXIT;
8407 }
8408 }
8409
8410 if (pVCpu->hm.s.svm.fEmulateLongModeSysEnterExit)
8411 {
8412 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_SS | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS
8413 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);
8414 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8415 {
8416 /* Ideally, IEM should just handle all these special #UD situations, but
8417 we don't quite trust things to behave optimially when doing that. So,
8418 for now we'll restrict ourselves to a handful of possible sysenter and
8419 sysexit encodings that we filter right here. */
8420 uint8_t abInstr[SVM_CTRL_GUEST_INSTR_BYTES_MAX];
8421 uint8_t cbInstr = pVmcb->ctrl.cbInstrFetched;
8422 uint32_t const uCpl = CPUMGetGuestCPL(pVCpu);
8423 uint8_t const cbMin = uCpl != 0 ? 2 : 1 + 2;
8424 RTGCPTR const GCPtrInstr = pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base;
8425 if (cbInstr < cbMin || cbInstr > SVM_CTRL_GUEST_INSTR_BYTES_MAX)
8426 {
8427 cbInstr = cbMin;
8428 int rc2 = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, GCPtrInstr, cbInstr);
8429 AssertRCStmt(rc2, cbInstr = 0);
8430 }
8431 else
8432 memcpy(abInstr, pVmcb->ctrl.abInstr, cbInstr); /* unlikely */
8433 if ( cbInstr == 0 /* read error */
8434 || (cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x34) /* sysenter */
8435 || ( uCpl == 0
8436 && ( ( cbInstr >= 2 && abInstr[0] == 0x0f && abInstr[1] == 0x35) /* sysexit */
8437 || ( cbInstr >= 3 && abInstr[1] == 0x0f && abInstr[2] == 0x35 /* rex.w sysexit */
8438 && (abInstr[0] & (X86_OP_REX_W | 0xf0)) == X86_OP_REX_W))))
8439 {
8440 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
8441 | CPUMCTX_EXTRN_SREG_MASK /* without ES+DS+GS the app will #GP later - go figure */);
8442 Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8443 rcStrict = IEMExecOneWithPrefetchedByPC(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), GCPtrInstr, abInstr, cbInstr);
8444 Log6(("hmR0SvmExitXcptUD: sysenter/sysexit: rcStrict=%Rrc %04x:%08RX64 %08RX64 %04x:%08RX64\n",
8445 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.rflags.u,
8446 pVCpu->cpum.GstCtx.ss.Sel, pVCpu->cpum.GstCtx.rsp));
8447 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8448 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK); /** @todo Lazy bird. */
8449 if (rcStrict == VINF_IEM_RAISED_XCPT)
8450 rcStrict = VINF_SUCCESS;
8451 return rcStrict;
8452 }
8453 Log6(("hmR0SvmExitXcptUD: not sysenter/sysexit: %.*Rhxs at %#llx CPL=%u\n", cbInstr, abInstr, GCPtrInstr, uCpl));
8454 }
8455 else
8456 Log6(("hmR0SvmExitXcptUD: not in long mode at %04x:%llx\n", pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
8457 }
8458
8459 /* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
8460 if (RT_FAILURE(rcStrict))
8461 {
8462 hmR0SvmSetPendingXcptUD(pVCpu);
8463 rcStrict = VINF_SUCCESS;
8464 }
8465
8466 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8467 return rcStrict;
8468}
8469
8470
8471/**
8472 * \#VMEXIT handler for math-fault exceptions (SVM_EXIT_XCPT_16).
8473 * Conditional \#VMEXIT.
8474 */
8475HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8476{
8477 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8478 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8479 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8480
8481 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8482 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8483
8484 /* Paranoia; Ensure we cannot be called as a result of event delivery. */
8485 Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
8486
8487 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8488
8489 if (!(pCtx->cr0 & X86_CR0_NE))
8490 {
8491 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8492 PDISSTATE pDis = &pVCpu->hmr0.s.svm.DisState;
8493 unsigned cbInstr;
8494 int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbInstr);
8495 if (RT_SUCCESS(rc))
8496 {
8497 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
8498 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13 /* u8Irq */, 1 /* u8Level */, 0 /* uTagSrc */);
8499 if (RT_SUCCESS(rc))
8500 hmR0SvmAdvanceRip(pVCpu, cbInstr);
8501 }
8502 else
8503 Log4Func(("EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
8504 return rc;
8505 }
8506
8507 hmR0SvmSetPendingXcptMF(pVCpu);
8508 return VINF_SUCCESS;
8509}
8510
8511
8512/**
8513 * \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1). Conditional
8514 * \#VMEXIT.
8515 */
8516HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8517{
8518 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8519 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8520 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8521 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8522
8523 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
8524 {
8525 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
8526 return VINF_EM_RAW_INJECT_TRPM_EVENT;
8527 }
8528
8529 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8530
8531 /*
8532 * This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data
8533 * breakpoint). However, for both cases DR6 and DR7 are updated to what the exception
8534 * handler expects. See AMD spec. 15.12.2 "#DB (Debug)".
8535 */
8536 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8537 PSVMVMCB pVmcb = pVCpu->hmr0.s.svm.pVmcb;
8538 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8539 int rc = DBGFTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
8540 if (rc == VINF_EM_RAW_GUEST_TRAP)
8541 {
8542 Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
8543 if (CPUMIsHyperDebugStateActive(pVCpu))
8544 CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) | pVmcb->guest.u64DR6);
8545
8546 /* Reflect the exception back to the guest. */
8547 hmR0SvmSetPendingXcptDB(pVCpu);
8548 rc = VINF_SUCCESS;
8549 }
8550
8551 /*
8552 * Update DR6.
8553 */
8554 if (CPUMIsHyperDebugStateActive(pVCpu))
8555 {
8556 Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
8557 pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
8558 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
8559 }
8560 else
8561 {
8562 AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
8563 Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
8564 }
8565
8566 return rc;
8567}
8568
8569
8570/**
8571 * \#VMEXIT handler for alignment check exceptions (SVM_EXIT_XCPT_17).
8572 * Conditional \#VMEXIT.
8573 */
8574HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8575{
8576 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8577 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8578 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC);
8579
8580 SVMEVENT Event;
8581 Event.u = 0;
8582 Event.n.u1Valid = 1;
8583 Event.n.u3Type = SVM_EVENT_EXCEPTION;
8584 Event.n.u8Vector = X86_XCPT_AC;
8585 Event.n.u1ErrorCodeValid = 1;
8586 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8587 return VINF_SUCCESS;
8588}
8589
8590
8591/**
8592 * \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
8593 * Conditional \#VMEXIT.
8594 */
8595HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8596{
8597 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8598 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
8599 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8600 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8601
8602 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8603 VBOXSTRICTRC rc = DBGFTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
8604 if (rc == VINF_EM_RAW_GUEST_TRAP)
8605 {
8606 SVMEVENT Event;
8607 Event.u = 0;
8608 Event.n.u1Valid = 1;
8609 Event.n.u3Type = SVM_EVENT_EXCEPTION;
8610 Event.n.u8Vector = X86_XCPT_BP;
8611 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8612 rc = VINF_SUCCESS;
8613 }
8614
8615 Assert(rc == VINF_SUCCESS || rc == VINF_EM_DBG_BREAKPOINT);
8616 return rc;
8617}
8618
8619
8620/**
8621 * Hacks its way around the lovely mesa driver's backdoor accesses.
8622 *
8623 * @sa hmR0VmxHandleMesaDrvGp
8624 */
8625static int hmR0SvmHandleMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8626{
8627 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_GPRS_MASK);
8628 Log(("hmR0SvmHandleMesaDrvGp: at %04x:%08RX64 rcx=%RX64 rbx=%RX64\n",
8629 pVmcb->guest.CS.u16Sel, pVmcb->guest.u64RIP, pCtx->rcx, pCtx->rbx));
8630 RT_NOREF(pCtx, pVmcb);
8631
8632 /* For now we'll just skip the instruction. */
8633 hmR0SvmAdvanceRip(pVCpu, 1);
8634 return VINF_SUCCESS;
8635}
8636
8637
8638/**
8639 * Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
8640 * backdoor logging w/o checking what it is running inside.
8641 *
8642 * This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
8643 * backdoor port and magic numbers loaded in registers.
8644 *
8645 * @returns true if it is, false if it isn't.
8646 * @sa hmR0VmxIsMesaDrvGp
8647 */
8648DECLINLINE(bool) hmR0SvmIsMesaDrvGp(PVMCPUCC pVCpu, PCPUMCTX pCtx, PCSVMVMCB pVmcb)
8649{
8650 /* Check magic and port. */
8651 Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RDX | CPUMCTX_EXTRN_RCX)));
8652 /*Log8(("hmR0SvmIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax, pCtx->rdx));*/
8653 if (pCtx->dx != UINT32_C(0x5658))
8654 return false;
8655 if ((pCtx->fExtrn & CPUMCTX_EXTRN_RAX ? pVmcb->guest.u64RAX : pCtx->rax) != UINT32_C(0x564d5868))
8656 return false;
8657
8658 /* Check that it is #GP(0). */
8659 if (pVmcb->ctrl.u64ExitInfo1 != 0)
8660 return false;
8661
8662 /* Flat ring-3 CS. */
8663 /*Log8(("hmR0SvmIsMesaDrvGp: u8CPL=%d base=%RX64\n", pVmcb->guest.u8CPL, pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base));*/
8664 if (pVmcb->guest.u8CPL != 3)
8665 return false;
8666 if ((pCtx->fExtrn & CPUMCTX_EXTRN_CS ? pVmcb->guest.CS.u64Base : pCtx->cs.u64Base) != 0)
8667 return false;
8668
8669 /* 0xed: IN eAX,dx */
8670 if (pVmcb->ctrl.cbInstrFetched < 1) /* unlikely, it turns out. */
8671 {
8672 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_GPRS_MASK
8673 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_EFER);
8674 uint8_t abInstr[1];
8675 int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
8676 /*Log8(("hmR0SvmIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0])); */
8677 if (RT_FAILURE(rc))
8678 return false;
8679 if (abInstr[0] != 0xed)
8680 return false;
8681 }
8682 else
8683 {
8684 /*Log8(("hmR0SvmIsMesaDrvGp: %#x\n", pVmcb->ctrl.abInstr));*/
8685 if (pVmcb->ctrl.abInstr[0] != 0xed)
8686 return false;
8687 }
8688 return true;
8689}
8690
8691
8692/**
8693 * \#VMEXIT handler for general protection faults (SVM_EXIT_XCPT_BP).
8694 * Conditional \#VMEXIT.
8695 */
8696HMSVM_EXIT_DECL hmR0SvmExitXcptGP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8697{
8698 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8699 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8700 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8701
8702 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8703 Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8704
8705 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8706 if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
8707 || !hmR0SvmIsMesaDrvGp(pVCpu, pCtx, pVmcb))
8708 {
8709 SVMEVENT Event;
8710 Event.u = 0;
8711 Event.n.u1Valid = 1;
8712 Event.n.u3Type = SVM_EVENT_EXCEPTION;
8713 Event.n.u8Vector = X86_XCPT_GP;
8714 Event.n.u1ErrorCodeValid = 1;
8715 Event.n.u32ErrorCode = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
8716 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8717 return VINF_SUCCESS;
8718 }
8719 return hmR0SvmHandleMesaDrvGp(pVCpu, pCtx, pVmcb);
8720}
8721
8722
8723/**
8724 * \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
8725 */
8726HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8727{
8728 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8729 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8730
8731 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8732 uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_XCPT_0;
8733 uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
8734 Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
8735 Assert(uVector <= X86_XCPT_LAST);
8736 Log4Func(("uVector=%#x uErrCode=%u\n", uVector, uErrCode));
8737
8738 SVMEVENT Event;
8739 Event.u = 0;
8740 Event.n.u1Valid = 1;
8741 Event.n.u3Type = SVM_EVENT_EXCEPTION;
8742 Event.n.u8Vector = uVector;
8743 switch (uVector)
8744 {
8745 /* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
8746 case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
8747 case X86_XCPT_DF:
8748 case X86_XCPT_TS:
8749 case X86_XCPT_NP:
8750 case X86_XCPT_SS:
8751 case X86_XCPT_GP:
8752 case X86_XCPT_AC:
8753 {
8754 Event.n.u1ErrorCodeValid = 1;
8755 Event.n.u32ErrorCode = uErrCode;
8756 break;
8757 }
8758 }
8759
8760#ifdef VBOX_WITH_STATISTICS
8761 switch (uVector)
8762 {
8763 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
8764 case X86_XCPT_DB: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB); break;
8765 case X86_XCPT_BP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP); break;
8766 case X86_XCPT_OF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8767 case X86_XCPT_BR: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBR); break;
8768 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
8769 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestOF); break;
8770 case X86_XCPT_DF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDF); break;
8771 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS); break;
8772 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
8773 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
8774 case X86_XCPT_GP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP); break;
8775 case X86_XCPT_PF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF); break;
8776 case X86_XCPT_MF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF); break;
8777 case X86_XCPT_AC: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestAC); break;
8778 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
8779 default:
8780 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8781 break;
8782 }
8783#endif
8784
8785 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8786 return VINF_SUCCESS;
8787}
8788
8789
8790/**
8791 * \#VMEXIT handler for software interrupt (INTn). Conditional \#VMEXIT (debug).
8792 */
8793HMSVM_EXIT_DECL hmR0SvmExitSwInt(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8794{
8795 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8796 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
8797
8798 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8799 SVMEVENT Event;
8800 Event.u = 0;
8801 Event.n.u1Valid = 1;
8802 Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
8803 Event.n.u8Vector = pVmcb->ctrl.u64ExitInfo1 & 0xff;
8804 Log4Func(("uVector=%#x\n", Event.n.u8Vector));
8805 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
8806 return VINF_SUCCESS;
8807}
8808
8809
8810/**
8811 * Generic exit handler that interprets the current instruction
8812 *
8813 * Useful exit that only gets triggered by dtrace and the debugger. Caller does
8814 * the exit logging, and this function does the rest.
8815 */
8816static VBOXSTRICTRC hmR0SvmExitInterpretInstruction(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient,
8817 uint64_t fExtraImport, uint64_t fHmChanged)
8818{
8819#if 1
8820 RT_NOREF(pSvmTransient);
8821 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | fExtraImport);
8822 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
8823 if (rcStrict == VINF_SUCCESS)
8824 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RIP);
8825 else
8826 {
8827 Log4Func(("IEMExecOne -> %Rrc\n", VBOXSTRICTRC_VAL(rcStrict) ));
8828 if (rcStrict == VINF_IEM_RAISED_XCPT)
8829 {
8830 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK | fHmChanged);
8831 rcStrict = VINF_SUCCESS;
8832 }
8833 else
8834 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, fHmChanged);
8835 }
8836 return rcStrict;
8837#else
8838 RT_NOREF(pVCpu, pSvmTransient, fExtraImport, fHmChanged);
8839 return VINF_EM_RAW_EMULATE_INSTR;
8840#endif
8841}
8842
8843
8844/**
8845 * \#VMEXIT handler for STR. Conditional \#VMEXIT (debug).
8846 */
8847HMSVM_EXIT_DECL hmR0SvmExitTrRead(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8848{
8849 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8850 Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8851 return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR, 0);
8852}
8853
8854
8855/**
8856 * \#VMEXIT handler for LTR. Conditional \#VMEXIT (OS/2 TLB workaround, debug).
8857 */
8858HMSVM_EXIT_DECL hmR0SvmExitTrWrite(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8859{
8860 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8861
8862 /* Workaround for lack of TLB flushing in OS/2 when returning to protected
8863 mode after a real mode call (like a BIOS call). See ticketref:20625
8864 comment 14. */
8865 PVMCC pVM = pVCpu->CTX_SUFF(pVM);
8866 if (pVM->hm.s.fMissingOS2TlbFlushWorkaround)
8867 {
8868 Log4Func(("%04x:%08RX64 TLB flush\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8869 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
8870 }
8871 else
8872 Log4Func(("%04x:%08RX64\n", pSvmTransient->pVmcb->guest.CS.u16Sel, pSvmTransient->pVmcb->guest.u64RIP));
8873
8874 return hmR0SvmExitInterpretInstruction(pVCpu, pSvmTransient, CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_GDTR, HM_CHANGED_GUEST_TR);
8875}
8876
8877
8878#ifdef VBOX_WITH_NESTED_HWVIRT_SVM
8879/**
8880 * \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
8881 */
8882HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8883{
8884 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8885
8886 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8887 Assert(pVmcb);
8888 Assert(!pVmcb->ctrl.IntCtrl.n.u1VGifEnable);
8889
8890 VBOXSTRICTRC rcStrict;
8891 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8892 uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8893 if (fSupportsNextRipSave)
8894 {
8895 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | fImport);
8896 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8897 rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
8898 }
8899 else
8900 {
8901 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | fImport);
8902 rcStrict = IEMExecOne(pVCpu);
8903 }
8904
8905 if (rcStrict == VINF_SUCCESS)
8906 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8907 else if (rcStrict == VINF_IEM_RAISED_XCPT)
8908 {
8909 rcStrict = VINF_SUCCESS;
8910 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8911 }
8912 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8913 return rcStrict;
8914}
8915
8916
8917/**
8918 * \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
8919 */
8920HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8921{
8922 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8923
8924 /*
8925 * When VGIF is not used we always intercept STGI instructions. When VGIF is used,
8926 * we only intercept STGI when events are pending for GIF to become 1.
8927 */
8928 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8929 if (pVmcb->ctrl.IntCtrl.n.u1VGifEnable)
8930 hmR0SvmClearCtrlIntercept(pVCpu, pVmcb, SVM_CTRL_INTERCEPT_STGI);
8931
8932 VBOXSTRICTRC rcStrict;
8933 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8934 uint64_t const fImport = CPUMCTX_EXTRN_HWVIRT;
8935 if (fSupportsNextRipSave)
8936 {
8937 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | fImport);
8938 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8939 rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
8940 }
8941 else
8942 {
8943 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | fImport);
8944 rcStrict = IEMExecOne(pVCpu);
8945 }
8946
8947 if (rcStrict == VINF_SUCCESS)
8948 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_HWVIRT);
8949 else if (rcStrict == VINF_IEM_RAISED_XCPT)
8950 {
8951 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8952 rcStrict = VINF_SUCCESS;
8953 }
8954 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
8955 return rcStrict;
8956}
8957
8958
8959/**
8960 * \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
8961 */
8962HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
8963{
8964 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
8965
8966 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
8967 Assert(pVmcb);
8968 Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
8969
8970 VBOXSTRICTRC rcStrict;
8971 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
8972 uint64_t const fImport = CPUMCTX_EXTRN_FS | CPUMCTX_EXTRN_GS | CPUMCTX_EXTRN_KERNEL_GS_BASE
8973 | CPUMCTX_EXTRN_TR | CPUMCTX_EXTRN_LDTR | CPUMCTX_EXTRN_SYSCALL_MSRS
8974 | CPUMCTX_EXTRN_SYSENTER_MSRS;
8975 if (fSupportsNextRipSave)
8976 {
8977 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | fImport);
8978 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
8979 rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
8980 }
8981 else
8982 {
8983 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | fImport);
8984 rcStrict = IEMExecOne(pVCpu);
8985 }
8986
8987 if (rcStrict == VINF_SUCCESS)
8988 {
8989 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS | HM_CHANGED_GUEST_GS
8990 | HM_CHANGED_GUEST_TR | HM_CHANGED_GUEST_LDTR
8991 | HM_CHANGED_GUEST_KERNEL_GS_BASE | HM_CHANGED_GUEST_SYSCALL_MSRS
8992 | HM_CHANGED_GUEST_SYSENTER_MSR_MASK);
8993 }
8994 else if (rcStrict == VINF_IEM_RAISED_XCPT)
8995 {
8996 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
8997 rcStrict = VINF_SUCCESS;
8998 }
8999 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9000 return rcStrict;
9001}
9002
9003
9004/**
9005 * \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
9006 */
9007HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9008{
9009 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9010
9011 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9012 Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
9013
9014 VBOXSTRICTRC rcStrict;
9015 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9016 if (fSupportsNextRipSave)
9017 {
9018 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9019 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9020 rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
9021 }
9022 else
9023 {
9024 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9025 rcStrict = IEMExecOne(pVCpu);
9026 }
9027
9028 if (rcStrict == VINF_IEM_RAISED_XCPT)
9029 {
9030 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9031 rcStrict = VINF_SUCCESS;
9032 }
9033 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9034 return rcStrict;
9035}
9036
9037
9038/**
9039 * \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
9040 */
9041HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9042{
9043 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9044
9045 VBOXSTRICTRC rcStrict;
9046 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9047 if (fSupportsNextRipSave)
9048 {
9049 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
9050 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9051 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9052 rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
9053 }
9054 else
9055 {
9056 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
9057 rcStrict = IEMExecOne(pVCpu);
9058 }
9059
9060 if (rcStrict == VINF_IEM_RAISED_XCPT)
9061 {
9062 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9063 rcStrict = VINF_SUCCESS;
9064 }
9065 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9066 return rcStrict;
9067}
9068
9069
9070/**
9071 * \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
9072 */
9073HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9074{
9075 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9076 /* We shall import the entire state here, just in case we enter and continue execution of
9077 the nested-guest with hardware-assisted SVM in ring-0, we would be switching VMCBs and
9078 could lose lose part of CPU state. */
9079 HMSVM_CPUMCTX_IMPORT_STATE(pVCpu, HMSVM_CPUMCTX_EXTRN_ALL);
9080
9081 VBOXSTRICTRC rcStrict;
9082 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu);
9083 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitVmentry, z);
9084 if (fSupportsNextRipSave)
9085 {
9086 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu);
9087 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pVCpu->cpum.GstCtx.rip;
9088 rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
9089 }
9090 else
9091 {
9092 /* We use IEMExecOneBypassEx() here as it supresses attempt to continue emulating any
9093 instruction(s) when interrupt inhibition is set as part of emulating the VMRUN
9094 instruction itself, see @bugref{7243#c126} */
9095 rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(&pVCpu->cpum.GstCtx), NULL /* pcbWritten */);
9096 }
9097 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitVmentry, z);
9098
9099 if (rcStrict == VINF_SUCCESS)
9100 {
9101 rcStrict = VINF_SVM_VMRUN;
9102 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_SVM_VMRUN_MASK);
9103 }
9104 else if (rcStrict == VINF_IEM_RAISED_XCPT)
9105 {
9106 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
9107 rcStrict = VINF_SUCCESS;
9108 }
9109 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
9110 return rcStrict;
9111}
9112
9113
9114/**
9115 * Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_XCPT_1).
9116 * Unconditional \#VMEXIT.
9117 */
9118HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9119{
9120 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9121 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9122
9123 if (pVCpu->hm.s.Event.fPending)
9124 {
9125 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterpret);
9126 return VINF_EM_RAW_INJECT_TRPM_EVENT;
9127 }
9128
9129 hmR0SvmSetPendingXcptDB(pVCpu);
9130 return VINF_SUCCESS;
9131}
9132
9133
9134/**
9135 * Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_XCPT_3).
9136 * Conditional \#VMEXIT.
9137 */
9138HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPUCC pVCpu, PSVMTRANSIENT pSvmTransient)
9139{
9140 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pSvmTransient);
9141 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY(pVCpu, pSvmTransient);
9142
9143 SVMEVENT Event;
9144 Event.u = 0;
9145 Event.n.u1Valid = 1;
9146 Event.n.u3Type = SVM_EVENT_EXCEPTION;
9147 Event.n.u8Vector = X86_XCPT_BP;
9148 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
9149 return VINF_SUCCESS;
9150}
9151#endif /* VBOX_WITH_NESTED_HWVIRT_SVM */
9152
9153/** @} */
9154
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette