VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMSVMR0.cpp@ 70413

Last change on this file since 70413 was 70413, checked in by vboxsync, 7 years ago

VMM/HM: Nested Hw.virt: Implemented virtualized VMSAVE/VMLOAD to avoid intercepting them when supported by the CPU.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 300.8 KB
Line 
1/* $Id: HMSVMR0.cpp 70413 2018-01-02 07:22:26Z vboxsync $ */
2/** @file
3 * HM SVM (AMD-V) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2013-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/tm.h>
32#include <VBox/vmm/gim.h>
33#include <VBox/vmm/apic.h>
34#include "HMInternal.h"
35#include <VBox/vmm/vm.h>
36#include "HMSVMR0.h"
37#include "dtrace/VBoxVMM.h"
38
39#define HMSVM_USE_IEM_EVENT_REFLECTION
40#ifdef DEBUG_ramshankar
41# define HMSVM_SYNC_FULL_GUEST_STATE
42# define HMSVM_ALWAYS_TRAP_ALL_XCPTS
43# define HMSVM_ALWAYS_TRAP_PF
44# define HMSVM_ALWAYS_TRAP_TASK_SWITCH
45#endif
46
47
48/*********************************************************************************************************************************
49* Defined Constants And Macros *
50*********************************************************************************************************************************/
51#ifdef VBOX_WITH_STATISTICS
52# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { \
53 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll); \
54 if ((u64ExitCode) == SVM_EXIT_NPF) \
55 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf); \
56 else \
57 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[(u64ExitCode) & MASK_EXITREASON_STAT]); \
58 } while (0)
59#else
60# define HMSVM_EXITCODE_STAM_COUNTER_INC(u64ExitCode) do { } while (0)
61#endif
62
63/** If we decide to use a function table approach this can be useful to
64 * switch to a "static DECLCALLBACK(int)". */
65#define HMSVM_EXIT_DECL static int
66
67/** Macro for checking and returning from the using function for
68 * \#VMEXIT intercepts that maybe caused during delivering of another
69 * event in the guest. */
70#ifdef VBOX_WITH_NESTED_HWVIRT
71# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY() \
72 do \
73 { \
74 int rc = hmR0SvmCheckExitDueToEventDelivery(pVCpu, pCtx, pSvmTransient); \
75 if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
76 else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
77 else if ( rc == VINF_EM_RESET \
78 && HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN)) \
79 return VBOXSTRICTRC_TODO(IEMExecSvmVmexit(pVCpu, SVM_EXIT_SHUTDOWN, 0, 0)); \
80 else \
81 return rc; \
82 } while (0)
83#else
84# define HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY() \
85 do \
86 { \
87 int rc = hmR0SvmCheckExitDueToEventDelivery(pVCpu, pCtx, pSvmTransient); \
88 if (RT_LIKELY(rc == VINF_SUCCESS)) { /* continue #VMEXIT handling */ } \
89 else if ( rc == VINF_HM_DOUBLE_FAULT) { return VINF_SUCCESS; } \
90 else \
91 return rc; \
92 } while (0)
93#endif
94
95/**
96 * Updates interrupt shadow for the current RIP.
97 */
98#define HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx) \
99 do { \
100 /* Update interrupt shadow. */ \
101 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS) \
102 && pCtx->rip != EMGetInhibitInterruptsPC(pVCpu)) \
103 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS); \
104 } while (0)
105
106/** Macro for upgrading a @a a_rc to VINF_EM_DBG_STEPPED after emulating an
107 * instruction that exited. */
108#define HMSVM_CHECK_SINGLE_STEP(a_pVCpu, a_rc) \
109 do { \
110 if ((a_pVCpu)->hm.s.fSingleInstruction && (a_rc) == VINF_SUCCESS) \
111 (a_rc) = VINF_EM_DBG_STEPPED; \
112 } while (0)
113
114/** Assert that preemption is disabled or covered by thread-context hooks. */
115#define HMSVM_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
116 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
117
118/** Assert that we haven't migrated CPUs when thread-context hooks are not
119 * used. */
120#define HMSVM_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
121 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
122 ("Illegal migration! Entered on CPU %u Current %u\n", \
123 pVCpu->hm.s.idEnteredCpu, RTMpCpuId()));
124
125/** Assert that we're not executing a nested-guest. */
126#ifdef VBOX_WITH_NESTED_HWVIRT
127# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) Assert(!CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
128#else
129# define HMSVM_ASSERT_NOT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
130#endif
131
132/** Assert that we're executing a nested-guest. */
133#ifdef VBOX_WITH_NESTED_HWVIRT
134# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) Assert(CPUMIsGuestInSvmNestedHwVirtMode((a_pCtx)))
135#else
136# define HMSVM_ASSERT_IN_NESTED_GUEST(a_pCtx) do { NOREF((a_pCtx)); } while (0)
137#endif
138
139/**
140 * Exception bitmap mask for all contributory exceptions.
141 *
142 * Page fault is deliberately excluded here as it's conditional as to whether
143 * it's contributory or benign. Page faults are handled separately.
144 */
145#define HMSVM_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
146 | RT_BIT(X86_XCPT_DE))
147
148/**
149 * Mandatory/unconditional guest control intercepts.
150 *
151 * SMIs can and do happen in normal operation. We need not intercept them
152 * while executing the guest or nested-guest.
153 */
154#define HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS ( SVM_CTRL_INTERCEPT_INTR \
155 | SVM_CTRL_INTERCEPT_NMI \
156 | SVM_CTRL_INTERCEPT_INIT \
157 | SVM_CTRL_INTERCEPT_RDPMC \
158 | SVM_CTRL_INTERCEPT_CPUID \
159 | SVM_CTRL_INTERCEPT_RSM \
160 | SVM_CTRL_INTERCEPT_HLT \
161 | SVM_CTRL_INTERCEPT_IOIO_PROT \
162 | SVM_CTRL_INTERCEPT_MSR_PROT \
163 | SVM_CTRL_INTERCEPT_INVLPGA \
164 | SVM_CTRL_INTERCEPT_SHUTDOWN \
165 | SVM_CTRL_INTERCEPT_FERR_FREEZE \
166 | SVM_CTRL_INTERCEPT_VMRUN \
167 | SVM_CTRL_INTERCEPT_VMMCALL \
168 | SVM_CTRL_INTERCEPT_STGI \
169 | SVM_CTRL_INTERCEPT_CLGI \
170 | SVM_CTRL_INTERCEPT_SKINIT \
171 | SVM_CTRL_INTERCEPT_WBINVD \
172 | SVM_CTRL_INTERCEPT_MONITOR \
173 | SVM_CTRL_INTERCEPT_MWAIT \
174 | SVM_CTRL_INTERCEPT_XSETBV)
175
176/** @name VMCB Clean Bits.
177 *
178 * These flags are used for VMCB-state caching. A set VMCB Clean bit indicates
179 * AMD-V doesn't need to reload the corresponding value(s) from the VMCB in
180 * memory.
181 *
182 * @{ */
183/** All intercepts vectors, TSC offset, PAUSE filter counter. */
184#define HMSVM_VMCB_CLEAN_INTERCEPTS RT_BIT(0)
185/** I/O permission bitmap, MSR permission bitmap. */
186#define HMSVM_VMCB_CLEAN_IOPM_MSRPM RT_BIT(1)
187/** ASID. */
188#define HMSVM_VMCB_CLEAN_ASID RT_BIT(2)
189/** TRP: V_TPR, V_IRQ, V_INTR_PRIO, V_IGN_TPR, V_INTR_MASKING,
190V_INTR_VECTOR. */
191#define HMSVM_VMCB_CLEAN_TPR RT_BIT(3)
192/** Nested Paging: Nested CR3 (nCR3), PAT. */
193#define HMSVM_VMCB_CLEAN_NP RT_BIT(4)
194/** Control registers (CR0, CR3, CR4, EFER). */
195#define HMSVM_VMCB_CLEAN_CRX_EFER RT_BIT(5)
196/** Debug registers (DR6, DR7). */
197#define HMSVM_VMCB_CLEAN_DRX RT_BIT(6)
198/** GDT, IDT limit and base. */
199#define HMSVM_VMCB_CLEAN_DT RT_BIT(7)
200/** Segment register: CS, SS, DS, ES limit and base. */
201#define HMSVM_VMCB_CLEAN_SEG RT_BIT(8)
202/** CR2.*/
203#define HMSVM_VMCB_CLEAN_CR2 RT_BIT(9)
204/** Last-branch record (DbgCtlMsr, br_from, br_to, lastint_from, lastint_to) */
205#define HMSVM_VMCB_CLEAN_LBR RT_BIT(10)
206/** AVIC (AVIC APIC_BAR; AVIC APIC_BACKING_PAGE, AVIC
207PHYSICAL_TABLE and AVIC LOGICAL_TABLE Pointers). */
208#define HMSVM_VMCB_CLEAN_AVIC RT_BIT(11)
209/** Mask of all valid VMCB Clean bits. */
210#define HMSVM_VMCB_CLEAN_ALL ( HMSVM_VMCB_CLEAN_INTERCEPTS \
211 | HMSVM_VMCB_CLEAN_IOPM_MSRPM \
212 | HMSVM_VMCB_CLEAN_ASID \
213 | HMSVM_VMCB_CLEAN_TPR \
214 | HMSVM_VMCB_CLEAN_NP \
215 | HMSVM_VMCB_CLEAN_CRX_EFER \
216 | HMSVM_VMCB_CLEAN_DRX \
217 | HMSVM_VMCB_CLEAN_DT \
218 | HMSVM_VMCB_CLEAN_SEG \
219 | HMSVM_VMCB_CLEAN_CR2 \
220 | HMSVM_VMCB_CLEAN_LBR \
221 | HMSVM_VMCB_CLEAN_AVIC)
222/** @} */
223
224/** @name SVM transient.
225 *
226 * A state structure for holding miscellaneous information across AMD-V
227 * VMRUN/\#VMEXIT operation, restored after the transition.
228 *
229 * @{ */
230typedef struct SVMTRANSIENT
231{
232 /** The host's rflags/eflags. */
233 RTCCUINTREG fEFlags;
234#if HC_ARCH_BITS == 32
235 uint32_t u32Alignment0;
236#endif
237
238 /** The \#VMEXIT exit code (the EXITCODE field in the VMCB). */
239 uint64_t u64ExitCode;
240 /** The guest's TPR value used for TPR shadowing. */
241 uint8_t u8GuestTpr;
242 /** Alignment. */
243 uint8_t abAlignment0[7];
244
245 /** Whether the guest FPU state was active at the time of \#VMEXIT. */
246 bool fWasGuestFPUStateActive;
247 /** Whether the guest debug state was active at the time of \#VMEXIT. */
248 bool fWasGuestDebugStateActive;
249 /** Whether the hyper debug state was active at the time of \#VMEXIT. */
250 bool fWasHyperDebugStateActive;
251 /** Whether the TSC offset mode needs to be updated. */
252 bool fUpdateTscOffsetting;
253 /** Whether the TSC_AUX MSR needs restoring on \#VMEXIT. */
254 bool fRestoreTscAuxMsr;
255 /** Whether the \#VMEXIT was caused by a page-fault during delivery of a
256 * contributary exception or a page-fault. */
257 bool fVectoringDoublePF;
258 /** Whether the \#VMEXIT was caused by a page-fault during delivery of an
259 * external interrupt or NMI. */
260 bool fVectoringPF;
261} SVMTRANSIENT, *PSVMTRANSIENT;
262AssertCompileMemberAlignment(SVMTRANSIENT, u64ExitCode, sizeof(uint64_t));
263AssertCompileMemberAlignment(SVMTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
264/** @} */
265
266/**
267 * MSRPM (MSR permission bitmap) read permissions (for guest RDMSR).
268 */
269typedef enum SVMMSREXITREAD
270{
271 /** Reading this MSR causes a \#VMEXIT. */
272 SVMMSREXIT_INTERCEPT_READ = 0xb,
273 /** Reading this MSR does not cause a \#VMEXIT. */
274 SVMMSREXIT_PASSTHRU_READ
275} SVMMSREXITREAD;
276
277/**
278 * MSRPM (MSR permission bitmap) write permissions (for guest WRMSR).
279 */
280typedef enum SVMMSREXITWRITE
281{
282 /** Writing to this MSR causes a \#VMEXIT. */
283 SVMMSREXIT_INTERCEPT_WRITE = 0xd,
284 /** Writing to this MSR does not cause a \#VMEXIT. */
285 SVMMSREXIT_PASSTHRU_WRITE
286} SVMMSREXITWRITE;
287
288/**
289 * SVM \#VMEXIT handler.
290 *
291 * @returns VBox status code.
292 * @param pVCpu The cross context virtual CPU structure.
293 * @param pMixedCtx Pointer to the guest-CPU context.
294 * @param pSvmTransient Pointer to the SVM-transient structure.
295 */
296typedef int FNSVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient);
297
298
299/*********************************************************************************************************************************
300* Internal Functions *
301*********************************************************************************************************************************/
302static void hmR0SvmSetMsrPermission(PSVMVMCB pVmcb, uint8_t *pbMsrBitmap, unsigned uMsr, SVMMSREXITREAD enmRead,
303 SVMMSREXITWRITE enmWrite);
304static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu);
305static void hmR0SvmLeave(PVMCPU pVCpu);
306
307/** @name \#VMEXIT handlers.
308 * @{
309 */
310static FNSVMEXITHANDLER hmR0SvmExitIntr;
311static FNSVMEXITHANDLER hmR0SvmExitWbinvd;
312static FNSVMEXITHANDLER hmR0SvmExitInvd;
313static FNSVMEXITHANDLER hmR0SvmExitCpuid;
314static FNSVMEXITHANDLER hmR0SvmExitRdtsc;
315static FNSVMEXITHANDLER hmR0SvmExitRdtscp;
316static FNSVMEXITHANDLER hmR0SvmExitRdpmc;
317static FNSVMEXITHANDLER hmR0SvmExitInvlpg;
318static FNSVMEXITHANDLER hmR0SvmExitHlt;
319static FNSVMEXITHANDLER hmR0SvmExitMonitor;
320static FNSVMEXITHANDLER hmR0SvmExitMwait;
321static FNSVMEXITHANDLER hmR0SvmExitShutdown;
322static FNSVMEXITHANDLER hmR0SvmExitUnexpected;
323static FNSVMEXITHANDLER hmR0SvmExitReadCRx;
324static FNSVMEXITHANDLER hmR0SvmExitWriteCRx;
325static FNSVMEXITHANDLER hmR0SvmExitMsr;
326static FNSVMEXITHANDLER hmR0SvmExitReadDRx;
327static FNSVMEXITHANDLER hmR0SvmExitWriteDRx;
328static FNSVMEXITHANDLER hmR0SvmExitXsetbv;
329static FNSVMEXITHANDLER hmR0SvmExitIOInstr;
330static FNSVMEXITHANDLER hmR0SvmExitNestedPF;
331static FNSVMEXITHANDLER hmR0SvmExitVIntr;
332static FNSVMEXITHANDLER hmR0SvmExitTaskSwitch;
333static FNSVMEXITHANDLER hmR0SvmExitVmmCall;
334static FNSVMEXITHANDLER hmR0SvmExitPause;
335static FNSVMEXITHANDLER hmR0SvmExitIret;
336static FNSVMEXITHANDLER hmR0SvmExitXcptPF;
337static FNSVMEXITHANDLER hmR0SvmExitXcptNM;
338static FNSVMEXITHANDLER hmR0SvmExitXcptUD;
339static FNSVMEXITHANDLER hmR0SvmExitXcptMF;
340static FNSVMEXITHANDLER hmR0SvmExitXcptDB;
341static FNSVMEXITHANDLER hmR0SvmExitXcptAC;
342static FNSVMEXITHANDLER hmR0SvmExitXcptBP;
343#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) || defined(VBOX_WITH_NESTED_HWVIRT)
344static FNSVMEXITHANDLER hmR0SvmExitXcptGeneric;
345#endif
346#ifdef VBOX_WITH_NESTED_HWVIRT
347static FNSVMEXITHANDLER hmR0SvmExitXcptPFNested;
348static FNSVMEXITHANDLER hmR0SvmExitClgi;
349static FNSVMEXITHANDLER hmR0SvmExitStgi;
350static FNSVMEXITHANDLER hmR0SvmExitVmload;
351static FNSVMEXITHANDLER hmR0SvmExitVmsave;
352static FNSVMEXITHANDLER hmR0SvmExitInvlpga;
353static FNSVMEXITHANDLER hmR0SvmExitVmrun;
354static FNSVMEXITHANDLER hmR0SvmNestedExitXcptDB;
355static FNSVMEXITHANDLER hmR0SvmNestedExitXcptBP;
356#endif
357/** @} */
358
359static int hmR0SvmHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient);
360#ifdef VBOX_WITH_NESTED_HWVIRT
361static int hmR0SvmHandleExitNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient);
362#endif
363
364
365/*********************************************************************************************************************************
366* Global Variables *
367*********************************************************************************************************************************/
368/** Ring-0 memory object for the IO bitmap. */
369RTR0MEMOBJ g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
370/** Physical address of the IO bitmap. */
371RTHCPHYS g_HCPhysIOBitmap = 0;
372/** Pointer to the IO bitmap. */
373R0PTRTYPE(void *) g_pvIOBitmap = NULL;
374
375#ifdef VBOX_WITH_NESTED_HWVIRT
376/** Ring-0 memory object for the nested-guest MSRPM bitmap. */
377RTR0MEMOBJ g_hMemObjNstGstMsrBitmap = NIL_RTR0MEMOBJ;
378/** Physical address of the nested-guest MSRPM bitmap. */
379RTHCPHYS g_HCPhysNstGstMsrBitmap = 0;
380/** Pointer to the nested-guest MSRPM bitmap. */
381R0PTRTYPE(void *) g_pvNstGstMsrBitmap = NULL;
382#endif
383
384/**
385 * Sets up and activates AMD-V on the current CPU.
386 *
387 * @returns VBox status code.
388 * @param pCpu Pointer to the CPU info struct.
389 * @param pVM The cross context VM structure. Can be
390 * NULL after a resume!
391 * @param pvCpuPage Pointer to the global CPU page.
392 * @param HCPhysCpuPage Physical address of the global CPU page.
393 * @param fEnabledByHost Whether the host OS has already initialized AMD-V.
394 * @param pvArg Unused on AMD-V.
395 */
396VMMR0DECL(int) SVMR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
397 void *pvArg)
398{
399 Assert(!fEnabledByHost);
400 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
401 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
402 Assert(pvCpuPage); NOREF(pvCpuPage);
403 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
404
405 NOREF(pvArg);
406 NOREF(fEnabledByHost);
407
408 /* Paranoid: Disable interrupt as, in theory, interrupt handlers might mess with EFER. */
409 RTCCUINTREG fEFlags = ASMIntDisableFlags();
410
411 /*
412 * We must turn on AMD-V and setup the host state physical address, as those MSRs are per CPU.
413 */
414 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
415 if (u64HostEfer & MSR_K6_EFER_SVME)
416 {
417 /* If the VBOX_HWVIRTEX_IGNORE_SVM_IN_USE is active, then we blindly use AMD-V. */
418 if ( pVM
419 && pVM->hm.s.svm.fIgnoreInUseError)
420 {
421 pCpu->fIgnoreAMDVInUseError = true;
422 }
423
424 if (!pCpu->fIgnoreAMDVInUseError)
425 {
426 ASMSetFlags(fEFlags);
427 return VERR_SVM_IN_USE;
428 }
429 }
430
431 /* Turn on AMD-V in the EFER MSR. */
432 ASMWrMsr(MSR_K6_EFER, u64HostEfer | MSR_K6_EFER_SVME);
433
434 /* Write the physical page address where the CPU will store the host state while executing the VM. */
435 ASMWrMsr(MSR_K8_VM_HSAVE_PA, HCPhysCpuPage);
436
437 /* Restore interrupts. */
438 ASMSetFlags(fEFlags);
439
440 /*
441 * Theoretically, other hypervisors may have used ASIDs, ideally we should flush all non-zero ASIDs
442 * when enabling SVM. AMD doesn't have an SVM instruction to flush all ASIDs (flushing is done
443 * upon VMRUN). Therefore, flag that we need to flush the TLB entirely with before executing any
444 * guest code.
445 */
446 pCpu->fFlushAsidBeforeUse = true;
447
448 /*
449 * Ensure each VCPU scheduled on this CPU gets a new ASID on resume. See @bugref{6255}.
450 */
451 ++pCpu->cTlbFlushes;
452
453 return VINF_SUCCESS;
454}
455
456
457/**
458 * Deactivates AMD-V on the current CPU.
459 *
460 * @returns VBox status code.
461 * @param pCpu Pointer to the CPU info struct.
462 * @param pvCpuPage Pointer to the global CPU page.
463 * @param HCPhysCpuPage Physical address of the global CPU page.
464 */
465VMMR0DECL(int) SVMR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
466{
467 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
468 AssertReturn( HCPhysCpuPage
469 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
470 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
471 NOREF(pCpu);
472
473 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with EFER. */
474 RTCCUINTREG fEFlags = ASMIntDisableFlags();
475
476 /* Turn off AMD-V in the EFER MSR. */
477 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
478 ASMWrMsr(MSR_K6_EFER, u64HostEfer & ~MSR_K6_EFER_SVME);
479
480 /* Invalidate host state physical address. */
481 ASMWrMsr(MSR_K8_VM_HSAVE_PA, 0);
482
483 /* Restore interrupts. */
484 ASMSetFlags(fEFlags);
485
486 return VINF_SUCCESS;
487}
488
489
490/**
491 * Does global AMD-V initialization (called during module initialization).
492 *
493 * @returns VBox status code.
494 */
495VMMR0DECL(int) SVMR0GlobalInit(void)
496{
497 /*
498 * Allocate 12 KB for the IO bitmap. Since this is non-optional and we always intercept all IO accesses, it's done
499 * once globally here instead of per-VM.
500 */
501 Assert(g_hMemObjIOBitmap == NIL_RTR0MEMOBJ);
502 int rc = RTR0MemObjAllocCont(&g_hMemObjIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
503 if (RT_FAILURE(rc))
504 return rc;
505
506 g_pvIOBitmap = RTR0MemObjAddress(g_hMemObjIOBitmap);
507 g_HCPhysIOBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjIOBitmap, 0 /* iPage */);
508
509 /* Set all bits to intercept all IO accesses. */
510 ASMMemFill32(g_pvIOBitmap, SVM_IOPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
511
512#ifdef VBOX_WITH_NESTED_HWVIRT
513 /*
514 * Allocate 8 KB for the MSR permission bitmap for the nested-guest.
515 */
516 Assert(g_hMemObjNstGstMsrBitmap == NIL_RTR0MEMOBJ);
517 rc = RTR0MemObjAllocCont(&g_hMemObjNstGstMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, false /* fExecutable */);
518 if (RT_FAILURE(rc))
519 return rc;
520
521 g_pvNstGstMsrBitmap = RTR0MemObjAddress(g_hMemObjNstGstMsrBitmap);
522 g_HCPhysNstGstMsrBitmap = RTR0MemObjGetPagePhysAddr(g_hMemObjNstGstMsrBitmap, 0 /* iPage */);
523
524 /* Set all bits to intercept all MSR accesses. */
525 ASMMemFill32(g_pvNstGstMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
526#endif
527
528 return VINF_SUCCESS;
529}
530
531
532/**
533 * Does global AMD-V termination (called during module termination).
534 */
535VMMR0DECL(void) SVMR0GlobalTerm(void)
536{
537 if (g_hMemObjIOBitmap != NIL_RTR0MEMOBJ)
538 {
539 RTR0MemObjFree(g_hMemObjIOBitmap, true /* fFreeMappings */);
540 g_pvIOBitmap = NULL;
541 g_HCPhysIOBitmap = 0;
542 g_hMemObjIOBitmap = NIL_RTR0MEMOBJ;
543 }
544
545#ifdef VBOX_WITH_NESTED_HWVIRT
546 if (g_hMemObjNstGstMsrBitmap != NIL_RTR0MEMOBJ)
547 {
548 RTR0MemObjFree(g_hMemObjNstGstMsrBitmap, true /* fFreeMappings */);
549 g_pvNstGstMsrBitmap = NULL;
550 g_HCPhysNstGstMsrBitmap = 0;
551 g_hMemObjNstGstMsrBitmap = NIL_RTR0MEMOBJ;
552 }
553#endif
554}
555
556
557/**
558 * Frees any allocated per-VCPU structures for a VM.
559 *
560 * @param pVM The cross context VM structure.
561 */
562DECLINLINE(void) hmR0SvmFreeStructs(PVM pVM)
563{
564 for (uint32_t i = 0; i < pVM->cCpus; i++)
565 {
566 PVMCPU pVCpu = &pVM->aCpus[i];
567 AssertPtr(pVCpu);
568
569 if (pVCpu->hm.s.svm.hMemObjVmcbHost != NIL_RTR0MEMOBJ)
570 {
571 RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcbHost, false);
572 pVCpu->hm.s.svm.HCPhysVmcbHost = 0;
573 pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
574 }
575
576 if (pVCpu->hm.s.svm.hMemObjVmcb != NIL_RTR0MEMOBJ)
577 {
578 RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjVmcb, false);
579 pVCpu->hm.s.svm.pVmcb = NULL;
580 pVCpu->hm.s.svm.HCPhysVmcb = 0;
581 pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
582 }
583
584 if (pVCpu->hm.s.svm.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
585 {
586 RTR0MemObjFree(pVCpu->hm.s.svm.hMemObjMsrBitmap, false);
587 pVCpu->hm.s.svm.pvMsrBitmap = NULL;
588 pVCpu->hm.s.svm.HCPhysMsrBitmap = 0;
589 pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
590 }
591 }
592}
593
594
595/**
596 * Does per-VM AMD-V initialization.
597 *
598 * @returns VBox status code.
599 * @param pVM The cross context VM structure.
600 */
601VMMR0DECL(int) SVMR0InitVM(PVM pVM)
602{
603 int rc = VERR_INTERNAL_ERROR_5;
604
605 /*
606 * Check for an AMD CPU erratum which requires us to flush the TLB before every world-switch.
607 */
608 uint32_t u32Family;
609 uint32_t u32Model;
610 uint32_t u32Stepping;
611 if (HMAmdIsSubjectToErratum170(&u32Family, &u32Model, &u32Stepping))
612 {
613 Log4(("SVMR0InitVM: AMD cpu with erratum 170 family %#x model %#x stepping %#x\n", u32Family, u32Model, u32Stepping));
614 pVM->hm.s.svm.fAlwaysFlushTLB = true;
615 }
616
617 /*
618 * Initialize the R0 memory objects up-front so we can properly cleanup on allocation failures.
619 */
620 for (VMCPUID i = 0; i < pVM->cCpus; i++)
621 {
622 PVMCPU pVCpu = &pVM->aCpus[i];
623 pVCpu->hm.s.svm.hMemObjVmcbHost = NIL_RTR0MEMOBJ;
624 pVCpu->hm.s.svm.hMemObjVmcb = NIL_RTR0MEMOBJ;
625 pVCpu->hm.s.svm.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
626 }
627
628 for (VMCPUID i = 0; i < pVM->cCpus; i++)
629 {
630 PVMCPU pVCpu = &pVM->aCpus[i];
631
632 /*
633 * Allocate one page for the host-context VM control block (VMCB). This is used for additional host-state (such as
634 * FS, GS, Kernel GS Base, etc.) apart from the host-state save area specified in MSR_K8_VM_HSAVE_PA.
635 */
636 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcbHost, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
637 if (RT_FAILURE(rc))
638 goto failure_cleanup;
639
640 void *pvVmcbHost = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcbHost);
641 pVCpu->hm.s.svm.HCPhysVmcbHost = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcbHost, 0 /* iPage */);
642 Assert(pVCpu->hm.s.svm.HCPhysVmcbHost < _4G);
643 ASMMemZeroPage(pvVmcbHost);
644
645 /*
646 * Allocate one page for the guest-state VMCB.
647 */
648 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjVmcb, SVM_VMCB_PAGES << PAGE_SHIFT, false /* fExecutable */);
649 if (RT_FAILURE(rc))
650 goto failure_cleanup;
651
652 pVCpu->hm.s.svm.pVmcb = (PSVMVMCB)RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjVmcb);
653 pVCpu->hm.s.svm.HCPhysVmcb = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjVmcb, 0 /* iPage */);
654 Assert(pVCpu->hm.s.svm.HCPhysVmcb < _4G);
655 ASMMemZeroPage(pVCpu->hm.s.svm.pVmcb);
656
657 /*
658 * Allocate two pages (8 KB) for the MSR permission bitmap. There doesn't seem to be a way to convince
659 * SVM to not require one.
660 */
661 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.svm.hMemObjMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT,
662 false /* fExecutable */);
663 if (RT_FAILURE(rc))
664 goto failure_cleanup;
665
666 pVCpu->hm.s.svm.pvMsrBitmap = RTR0MemObjAddress(pVCpu->hm.s.svm.hMemObjMsrBitmap);
667 pVCpu->hm.s.svm.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.svm.hMemObjMsrBitmap, 0 /* iPage */);
668 /* Set all bits to intercept all MSR accesses (changed later on). */
669 ASMMemFill32(pVCpu->hm.s.svm.pvMsrBitmap, SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT, UINT32_C(0xffffffff));
670 }
671
672 return VINF_SUCCESS;
673
674failure_cleanup:
675 hmR0SvmFreeStructs(pVM);
676 return rc;
677}
678
679
680/**
681 * Does per-VM AMD-V termination.
682 *
683 * @returns VBox status code.
684 * @param pVM The cross context VM structure.
685 */
686VMMR0DECL(int) SVMR0TermVM(PVM pVM)
687{
688 hmR0SvmFreeStructs(pVM);
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Returns whether the VMCB Clean Bits feature is supported.
695 *
696 * @return @c true if supported, @c false otherwise.
697 * @param pVCpu The cross context virtual CPU structure.
698 * @param pCtx Pointer to the guest-CPU context.
699 */
700DECLINLINE(bool) hmR0SvmSupportsVmcbCleanBits(PVMCPU pVCpu, PCPUMCTX pCtx)
701{
702 PVM pVM = pVCpu->CTX_SUFF(pVM);
703#ifdef VBOX_WITH_NESTED_HWVIRT
704 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
705 {
706 return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN)
707 && pVM->cpum.ro.GuestFeatures.fSvmVmcbClean;
708 }
709#else
710 RT_NOREF(pCtx);
711#endif
712 return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VMCB_CLEAN);
713}
714
715
716/**
717 * Returns whether the decode assists feature is supported.
718 *
719 * @return @c true if supported, @c false otherwise.
720 * @param pVCpu The cross context virtual CPU structure.
721 * @param pCtx Pointer to the guest-CPU context.
722 */
723DECLINLINE(bool) hmR0SvmSupportsDecodeAssists(PVMCPU pVCpu, PCPUMCTX pCtx)
724{
725 PVM pVM = pVCpu->CTX_SUFF(pVM);
726#ifdef VBOX_WITH_NESTED_HWVIRT
727 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
728 {
729 return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS)
730 && pVM->cpum.ro.GuestFeatures.fSvmDecodeAssists;
731 }
732#else
733 RT_NOREF(pCtx);
734#endif
735 return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_DECODE_ASSISTS);
736}
737
738
739/**
740 * Returns whether the NRIP_SAVE feature is supported.
741 *
742 * @return @c true if supported, @c false otherwise.
743 * @param pVCpu The cross context virtual CPU structure.
744 * @param pCtx Pointer to the guest-CPU context.
745 */
746DECLINLINE(bool) hmR0SvmSupportsNextRipSave(PVMCPU pVCpu, PCPUMCTX pCtx)
747{
748 PVM pVM = pVCpu->CTX_SUFF(pVM);
749#ifdef VBOX_WITH_NESTED_HWVIRT
750 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
751 {
752 return (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE)
753 && pVM->cpum.ro.GuestFeatures.fSvmNextRipSave;
754 }
755#else
756 RT_NOREF(pCtx);
757#endif
758 return RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_NRIP_SAVE);
759}
760
761
762/**
763 * Sets the permission bits for the specified MSR in the MSRPM.
764 *
765 * @param pVmcb Pointer to the VM control block.
766 * @param pbMsrBitmap Pointer to the MSR bitmap.
767 * @param uMsr The MSR for which the access permissions are being set.
768 * @param enmRead MSR read permissions.
769 * @param enmWrite MSR write permissions.
770 */
771static void hmR0SvmSetMsrPermission(PSVMVMCB pVmcb, uint8_t *pbMsrBitmap, unsigned uMsr, SVMMSREXITREAD enmRead,
772 SVMMSREXITWRITE enmWrite)
773{
774 uint16_t offMsrpm;
775 uint32_t uMsrpmBit;
776 int rc = HMSvmGetMsrpmOffsetAndBit(uMsr, &offMsrpm, &uMsrpmBit);
777 AssertRC(rc);
778
779 Assert(uMsrpmBit < 0x3fff);
780 Assert(offMsrpm < SVM_MSRPM_PAGES << X86_PAGE_4K_SHIFT);
781
782 pbMsrBitmap += offMsrpm;
783 if (enmRead == SVMMSREXIT_INTERCEPT_READ)
784 ASMBitSet(pbMsrBitmap, uMsrpmBit);
785 else
786 ASMBitClear(pbMsrBitmap, uMsrpmBit);
787
788 if (enmWrite == SVMMSREXIT_INTERCEPT_WRITE)
789 ASMBitSet(pbMsrBitmap, uMsrpmBit + 1);
790 else
791 ASMBitClear(pbMsrBitmap, uMsrpmBit + 1);
792
793 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_IOPM_MSRPM;
794}
795
796
797/**
798 * Sets up AMD-V for the specified VM.
799 * This function is only called once per-VM during initalization.
800 *
801 * @returns VBox status code.
802 * @param pVM The cross context VM structure.
803 */
804VMMR0DECL(int) SVMR0SetupVM(PVM pVM)
805{
806 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
807 AssertReturn(pVM, VERR_INVALID_PARAMETER);
808 Assert(pVM->hm.s.svm.fSupported);
809
810 bool const fPauseFilter = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER);
811 bool const fPauseFilterThreshold = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_PAUSE_FILTER_THRESHOLD);
812 bool const fUsePauseFilter = fPauseFilter && pVM->hm.s.svm.cPauseFilter && pVM->hm.s.svm.cPauseFilterThresholdTicks;
813
814 bool const fLbrVirt = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_LBR_VIRT);
815 bool const fUseLbrVirt = fLbrVirt; /** @todo CFGM etc. */
816
817 bool const fVirtVmsaveVmload = RT_BOOL(pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_VIRT_VMSAVE_VMLOAD);
818 bool const fUseVirtVmsaveVmload = fVirtVmsaveVmload && pVM->hm.s.svm.fVirtVmsaveVmload && pVM->hm.s.fNestedPaging;
819
820 for (VMCPUID i = 0; i < pVM->cCpus; i++)
821 {
822 PVMCPU pVCpu = &pVM->aCpus[i];
823 PSVMVMCB pVmcb = pVM->aCpus[i].hm.s.svm.pVmcb;
824
825 AssertMsgReturn(pVmcb, ("Invalid pVmcb for vcpu[%u]\n", i), VERR_SVM_INVALID_PVMCB);
826
827 /* Initialize the #VMEXIT history array with end-of-array markers (UINT16_MAX). */
828 Assert(!pVCpu->hm.s.idxExitHistoryFree);
829 HMCPU_EXIT_HISTORY_RESET(pVCpu);
830
831 /* Always trap #AC for reasons of security. */
832 pVmcb->ctrl.u32InterceptXcpt |= RT_BIT_32(X86_XCPT_AC);
833
834 /* Always trap #DB for reasons of security. */
835 pVmcb->ctrl.u32InterceptXcpt |= RT_BIT_32(X86_XCPT_DB);
836
837 /* Trap exceptions unconditionally (debug purposes). */
838#ifdef HMSVM_ALWAYS_TRAP_PF
839 pVmcb->ctrl.u32InterceptXcpt |= RT_BIT(X86_XCPT_PF);
840#endif
841#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
842 /* If you add any exceptions here, make sure to update hmR0SvmHandleExit(). */
843 pVmcb->ctrl.u32InterceptXcpt |= 0
844 | RT_BIT(X86_XCPT_BP)
845 | RT_BIT(X86_XCPT_DE)
846 | RT_BIT(X86_XCPT_NM)
847 | RT_BIT(X86_XCPT_UD)
848 | RT_BIT(X86_XCPT_NP)
849 | RT_BIT(X86_XCPT_SS)
850 | RT_BIT(X86_XCPT_GP)
851 | RT_BIT(X86_XCPT_PF)
852 | RT_BIT(X86_XCPT_MF)
853 ;
854#endif
855
856 /* Set up unconditional intercepts and conditions. */
857 pVmcb->ctrl.u64InterceptCtrl = HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
858
859 /* CR0, CR4 reads must be intercepted, our shadow values are not necessarily the same as the guest's. */
860 pVmcb->ctrl.u16InterceptRdCRx = RT_BIT(0) | RT_BIT(4);
861
862 /* CR0, CR4 writes must be intercepted for the same reasons as above. */
863 pVmcb->ctrl.u16InterceptWrCRx = RT_BIT(0) | RT_BIT(4);
864
865 /* Intercept all DRx reads and writes by default. Changed later on. */
866 pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
867 pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
868
869 /* Virtualize masking of INTR interrupts. (reads/writes from/to CR8 go to the V_TPR register) */
870 pVmcb->ctrl.IntCtrl.n.u1VIntrMasking = 1;
871
872 /* Ignore the priority in the virtual TPR. This is necessary for delivering PIC style (ExtInt) interrupts
873 and we currently deliver both PIC and APIC interrupts alike. See hmR0SvmInjectPendingEvent() */
874 pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR = 1;
875
876 /* Set IO and MSR bitmap permission bitmap physical addresses. */
877 pVmcb->ctrl.u64IOPMPhysAddr = g_HCPhysIOBitmap;
878 pVmcb->ctrl.u64MSRPMPhysAddr = pVCpu->hm.s.svm.HCPhysMsrBitmap;
879
880 /* LBR virtualization. */
881 if (fUseLbrVirt)
882 {
883 pVmcb->ctrl.LbrVirt.n.u1LbrVirt = fUseLbrVirt;
884 pVmcb->guest.u64DBGCTL = MSR_IA32_DEBUGCTL_LBR;
885 }
886 else
887 Assert(pVmcb->ctrl.LbrVirt.n.u1LbrVirt == 0);
888
889 /* Virtualized VMSAVE/VMLOAD. */
890 pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload = fUseVirtVmsaveVmload;
891 if (!fUseVirtVmsaveVmload)
892 {
893 pVmcb->ctrl.u64InterceptCtrl |= SVM_CTRL_INTERCEPT_VMSAVE
894 | SVM_CTRL_INTERCEPT_VMLOAD;
895 }
896
897 /* Initially all VMCB clean bits MBZ indicating that everything should be loaded from the VMCB in memory. */
898 Assert(pVmcb->ctrl.u32VmcbCleanBits == 0);
899
900 /* The host ASID MBZ, for the guest start with 1. */
901 pVmcb->ctrl.TLBCtrl.n.u32ASID = 1;
902
903 /*
904 * Setup the PAT MSR (applicable for Nested Paging only).
905 * The default value should be 0x0007040600070406ULL, but we want to treat all guest memory as WB,
906 * so choose type 6 for all PAT slots.
907 */
908 pVmcb->guest.u64GPAT = UINT64_C(0x0006060606060606);
909
910 /* Setup Nested Paging. This doesn't change throughout the execution time of the VM. */
911 pVmcb->ctrl.NestedPaging.n.u1NestedPaging = pVM->hm.s.fNestedPaging;
912
913 /* Without Nested Paging, we need additionally intercepts. */
914 if (!pVM->hm.s.fNestedPaging)
915 {
916 /* CR3 reads/writes must be intercepted; our shadow values differ from the guest values. */
917 pVmcb->ctrl.u16InterceptRdCRx |= RT_BIT(3);
918 pVmcb->ctrl.u16InterceptWrCRx |= RT_BIT(3);
919
920 /* Intercept INVLPG and task switches (may change CR3, EFLAGS, LDT). */
921 pVmcb->ctrl.u64InterceptCtrl |= SVM_CTRL_INTERCEPT_INVLPG
922 | SVM_CTRL_INTERCEPT_TASK_SWITCH;
923
924 /* Page faults must be intercepted to implement shadow paging. */
925 pVmcb->ctrl.u32InterceptXcpt |= RT_BIT(X86_XCPT_PF);
926 }
927
928#ifdef HMSVM_ALWAYS_TRAP_TASK_SWITCH
929 pVmcb->ctrl.u64InterceptCtrl |= SVM_CTRL_INTERCEPT_TASK_SWITCH;
930#endif
931
932 /* Apply the exceptions intercepts needed by the GIM provider. */
933 if (pVCpu->hm.s.fGIMTrapXcptUD)
934 pVmcb->ctrl.u32InterceptXcpt |= RT_BIT(X86_XCPT_UD);
935
936 /* Setup Pause Filter for guest pause-loop (spinlock) exiting. */
937 if (fUsePauseFilter)
938 {
939 pVmcb->ctrl.u16PauseFilterCount = pVM->hm.s.svm.cPauseFilter;
940 if (fPauseFilterThreshold)
941 pVmcb->ctrl.u16PauseFilterThreshold = pVM->hm.s.svm.cPauseFilterThresholdTicks;
942 }
943
944 /*
945 * The following MSRs are saved/restored automatically during the world-switch.
946 * Don't intercept guest read/write accesses to these MSRs.
947 */
948 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
949 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
950 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_CSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
951 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K6_STAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
952 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_SF_MASK, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
953 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_FS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
954 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
955 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_KERNEL_GS_BASE, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
956 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_IA32_SYSENTER_CS, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
957 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_IA32_SYSENTER_ESP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
958 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_IA32_SYSENTER_EIP, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
959 }
960
961 return VINF_SUCCESS;
962}
963
964
965/**
966 * Gets a pointer to the currently active guest or nested-guest VMCB.
967 *
968 * @returns Pointer to the current context VMCB.
969 * @param pVCpu The cross context virtual CPU structure.
970 * @param pCtx Pointer to the guest-CPU context.
971 */
972DECLINLINE(PSVMVMCB) hmR0SvmGetCurrentVmcb(PVMCPU pVCpu, PCPUMCTX pCtx)
973{
974#ifdef VBOX_WITH_NESTED_HWVIRT
975 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
976 return pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
977#else
978 RT_NOREF(pCtx);
979#endif
980 return pVCpu->hm.s.svm.pVmcb;
981}
982
983
984/**
985 * Invalidates a guest page by guest virtual address.
986 *
987 * @returns VBox status code.
988 * @param pVM The cross context VM structure.
989 * @param pVCpu The cross context virtual CPU structure.
990 * @param GCVirt Guest virtual address of the page to invalidate.
991 */
992VMMR0DECL(int) SVMR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
993{
994 AssertReturn(pVM, VERR_INVALID_PARAMETER);
995 Assert(pVM->hm.s.svm.fSupported);
996
997 bool fFlushPending = pVM->hm.s.svm.fAlwaysFlushTLB || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
998
999 /* Skip it if a TLB flush is already pending. */
1000 if (!fFlushPending)
1001 {
1002 Log4(("SVMR0InvalidatePage %RGv\n", GCVirt));
1003
1004 PCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
1005 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
1006 AssertMsgReturn(pVmcb, ("Invalid pVmcb!\n"), VERR_SVM_INVALID_PVMCB);
1007
1008#if HC_ARCH_BITS == 32
1009 /* If we get a flush in 64-bit guest mode, then force a full TLB flush. INVLPGA takes only 32-bit addresses. */
1010 if (CPUMIsGuestInLongMode(pVCpu))
1011 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1012 else
1013#endif
1014 {
1015 SVMR0InvlpgA(GCVirt, pVmcb->ctrl.TLBCtrl.n.u32ASID);
1016 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1017 }
1018 }
1019 return VINF_SUCCESS;
1020}
1021
1022
1023/**
1024 * Flushes the appropriate tagged-TLB entries.
1025 *
1026 * @param pVCpu The cross context virtual CPU structure.
1027 * @param pCtx Pointer to the guest-CPU or nested-guest-CPU context.
1028 * @param pVmcb Pointer to the VM control block.
1029 */
1030static void hmR0SvmFlushTaggedTlb(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb)
1031{
1032#ifndef VBOX_WITH_NESTED_HWVIRT
1033 RT_NOREF(pCtx);
1034#endif
1035
1036 PVM pVM = pVCpu->CTX_SUFF(pVM);
1037 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
1038
1039 /*
1040 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1041 * This can happen both for start & resume due to long jumps back to ring-3.
1042 *
1043 * We also force a TLB flush every time when executing a nested-guest VCPU as there is no correlation
1044 * between it and the physical CPU.
1045 *
1046 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB,
1047 * so we cannot reuse the ASIDs without flushing.
1048 */
1049 bool fNewAsid = false;
1050 Assert(pCpu->idCpu != NIL_RTCPUID);
1051 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1052 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes
1053#ifdef VBOX_WITH_NESTED_HWVIRT
1054 || CPUMIsGuestInSvmNestedHwVirtMode(pCtx)
1055#endif
1056 )
1057 {
1058 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1059 pVCpu->hm.s.fForceTLBFlush = true;
1060 fNewAsid = true;
1061 }
1062
1063 /* Set TLB flush state as checked until we return from the world switch. */
1064 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
1065
1066 /* Check for explicit TLB flushes. */
1067 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1068 {
1069 pVCpu->hm.s.fForceTLBFlush = true;
1070 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1071 }
1072
1073 /*
1074 * If the AMD CPU erratum 170, We need to flush the entire TLB for each world switch. Sad.
1075 * This Host CPU requirement takes precedence.
1076 */
1077 if (pVM->hm.s.svm.fAlwaysFlushTLB)
1078 {
1079 pCpu->uCurrentAsid = 1;
1080 pVCpu->hm.s.uCurrentAsid = 1;
1081 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1082 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1083
1084 /* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1085 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1086
1087 /* Keep track of last CPU ID even when flushing all the time. */
1088 if (fNewAsid)
1089 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1090 }
1091 else
1092 {
1093 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_NOTHING;
1094 if (pVCpu->hm.s.fForceTLBFlush)
1095 {
1096 /* Clear the VMCB Clean Bit for NP while flushing the TLB. See @bugref{7152}. */
1097 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1098
1099 if (fNewAsid)
1100 {
1101 ++pCpu->uCurrentAsid;
1102
1103 bool fHitASIDLimit = false;
1104 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1105 {
1106 pCpu->uCurrentAsid = 1; /* Wraparound at 1; host uses 0 */
1107 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new ASID. */
1108 fHitASIDLimit = true;
1109 }
1110
1111 if ( fHitASIDLimit
1112 || pCpu->fFlushAsidBeforeUse)
1113 {
1114 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1115 pCpu->fFlushAsidBeforeUse = false;
1116 }
1117
1118 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1119 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1120 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1121 }
1122 else
1123 {
1124 if (pVM->hm.s.svm.u32Features & X86_CPUID_SVM_FEATURE_EDX_FLUSH_BY_ASID)
1125 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_SINGLE_CONTEXT;
1126 else
1127 pVmcb->ctrl.TLBCtrl.n.u8TLBFlush = SVM_TLB_FLUSH_ENTIRE;
1128 }
1129
1130 pVCpu->hm.s.fForceTLBFlush = false;
1131 }
1132 }
1133
1134 /* Update VMCB with the ASID. */
1135 if (pVmcb->ctrl.TLBCtrl.n.u32ASID != pVCpu->hm.s.uCurrentAsid)
1136 {
1137 pVmcb->ctrl.TLBCtrl.n.u32ASID = pVCpu->hm.s.uCurrentAsid;
1138 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_ASID;
1139 }
1140
1141 AssertMsg(pVCpu->hm.s.idLastCpu == pCpu->idCpu,
1142 ("vcpu idLastCpu=%u pcpu idCpu=%u\n", pVCpu->hm.s.idLastCpu, pCpu->idCpu));
1143 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1144 ("Flush count mismatch for cpu %u (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1145 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1146 ("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
1147 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1148 ("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1149
1150#ifdef VBOX_WITH_STATISTICS
1151 if (pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_NOTHING)
1152 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1153 else if ( pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT
1154 || pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_SINGLE_CONTEXT_RETAIN_GLOBALS)
1155 {
1156 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1157 }
1158 else
1159 {
1160 Assert(pVmcb->ctrl.TLBCtrl.n.u8TLBFlush == SVM_TLB_FLUSH_ENTIRE);
1161 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushEntire);
1162 }
1163#endif
1164}
1165
1166
1167/** @name 64-bit guest on 32-bit host OS helper functions.
1168 *
1169 * The host CPU is still 64-bit capable but the host OS is running in 32-bit
1170 * mode (code segment, paging). These wrappers/helpers perform the necessary
1171 * bits for the 32->64 switcher.
1172 *
1173 * @{ */
1174#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1175/**
1176 * Prepares for and executes VMRUN (64-bit guests on a 32-bit host).
1177 *
1178 * @returns VBox status code.
1179 * @param HCPhysVmcbHost Physical address of host VMCB.
1180 * @param HCPhysVmcb Physical address of the VMCB.
1181 * @param pCtx Pointer to the guest-CPU context.
1182 * @param pVM The cross context VM structure.
1183 * @param pVCpu The cross context virtual CPU structure.
1184 */
1185DECLASM(int) SVMR0VMSwitcherRun64(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu)
1186{
1187 uint32_t aParam[8];
1188 aParam[0] = RT_LO_U32(HCPhysVmcbHost); /* Param 1: HCPhysVmcbHost - Lo. */
1189 aParam[1] = RT_HI_U32(HCPhysVmcbHost); /* Param 1: HCPhysVmcbHost - Hi. */
1190 aParam[2] = RT_LO_U32(HCPhysVmcb); /* Param 2: HCPhysVmcb - Lo. */
1191 aParam[3] = RT_HI_U32(HCPhysVmcb); /* Param 2: HCPhysVmcb - Hi. */
1192 aParam[4] = VM_RC_ADDR(pVM, pVM);
1193 aParam[5] = 0;
1194 aParam[6] = VM_RC_ADDR(pVM, pVCpu);
1195 aParam[7] = 0;
1196
1197 return SVMR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_SVMRCVMRun64, RT_ELEMENTS(aParam), &aParam[0]);
1198}
1199
1200
1201/**
1202 * Executes the specified VMRUN handler in 64-bit mode.
1203 *
1204 * @returns VBox status code.
1205 * @param pVM The cross context VM structure.
1206 * @param pVCpu The cross context virtual CPU structure.
1207 * @param pCtx Pointer to the guest-CPU context.
1208 * @param enmOp The operation to perform.
1209 * @param cParams Number of parameters.
1210 * @param paParam Array of 32-bit parameters.
1211 */
1212VMMR0DECL(int) SVMR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
1213 uint32_t cParams, uint32_t *paParam)
1214{
1215 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
1216 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
1217
1218 NOREF(pCtx);
1219
1220 /* Disable interrupts. */
1221 RTHCUINTREG uOldEFlags = ASMIntDisableFlags();
1222
1223#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
1224 RTCPUID idHostCpu = RTMpCpuId();
1225 CPUMR0SetLApic(pVCpu, idHostCpu);
1226#endif
1227
1228 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
1229 CPUMSetHyperEIP(pVCpu, enmOp);
1230 for (int i = (int)cParams - 1; i >= 0; i--)
1231 CPUMPushHyper(pVCpu, paParam[i]);
1232
1233 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
1234 /* Call the switcher. */
1235 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
1236 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
1237
1238 /* Restore interrupts. */
1239 ASMSetFlags(uOldEFlags);
1240 return rc;
1241}
1242
1243#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
1244/** @} */
1245
1246
1247/**
1248 * Adds an exception to the intercept exception bitmap in the VMCB and updates
1249 * the corresponding VMCB Clean bit.
1250 *
1251 * @param pVmcb Pointer to the VM control block.
1252 * @param u32Xcpt The value of the exception (X86_XCPT_*).
1253 */
1254DECLINLINE(void) hmR0SvmAddXcptIntercept(PSVMVMCB pVmcb, uint32_t u32Xcpt)
1255{
1256 if (!(pVmcb->ctrl.u32InterceptXcpt & RT_BIT(u32Xcpt)))
1257 {
1258 pVmcb->ctrl.u32InterceptXcpt |= RT_BIT(u32Xcpt);
1259 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1260 }
1261}
1262
1263
1264/**
1265 * Removes an exception from the intercept-exception bitmap in the VMCB and
1266 * updates the corresponding VMCB Clean bit.
1267 *
1268 * @param pVCpu The cross context virtual CPU structure.
1269 * @param pCtx Pointer to the guest-CPU context.
1270 * @param pVmcb Pointer to the VM control block.
1271 * @param u32Xcpt The value of the exception (X86_XCPT_*).
1272 *
1273 * @remarks This takes into account if we're executing a nested-guest and only
1274 * removes the exception intercept if both the guest -and- nested-guest
1275 * are not intercepting it.
1276 */
1277DECLINLINE(void) hmR0SvmRemoveXcptIntercept(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb, uint32_t u32Xcpt)
1278{
1279 Assert(u32Xcpt != X86_XCPT_DB);
1280 Assert(u32Xcpt != X86_XCPT_AC);
1281#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
1282 if (pVmcb->ctrl.u32InterceptXcpt & RT_BIT(u32Xcpt))
1283 {
1284 bool fRemoveXcpt = true;
1285#ifdef VBOX_WITH_NESTED_HWVIRT
1286 /* Only remove the intercept if the nested-guest is also not intercepting it! */
1287 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
1288 {
1289 Assert(pCtx->hwvirt.svm.fHMCachedVmcb); NOREF(pCtx);
1290 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
1291 fRemoveXcpt = !(pVmcbNstGstCache->u32InterceptXcpt & RT_BIT(u32Xcpt));
1292 }
1293#else
1294 RT_NOREF2(pVCpu, pCtx);
1295#endif
1296 if (fRemoveXcpt)
1297 {
1298 pVmcb->ctrl.u32InterceptXcpt &= ~RT_BIT(u32Xcpt);
1299 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1300 }
1301 }
1302#else
1303 RT_NOREF3(pVCpu, pCtx, pVmcb);
1304#endif
1305}
1306
1307
1308/**
1309 * Loads the guest (or nested-guest) CR0 control register into the guest-state
1310 * area in the VMCB.
1311 *
1312 * Although the guest CR0 is a separate field in the VMCB we have to consider
1313 * the FPU state itself which is shared between the host and the guest.
1314 *
1315 * @returns VBox status code.
1316 * @param pVCpu The cross context virtual CPU structure.
1317 * @param pVmcb Pointer to the VM control block.
1318 * @param pCtx Pointer to the guest-CPU context.
1319 *
1320 * @remarks No-long-jump zone!!!
1321 */
1322static void hmR0SvmLoadSharedCR0(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1323{
1324 uint64_t u64GuestCR0 = pCtx->cr0;
1325
1326 /* Always enable caching. */
1327 u64GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW);
1328
1329 /*
1330 * When Nested Paging is not available use shadow page tables and intercept #PFs (the latter done in SVMR0SetupVM()).
1331 */
1332 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
1333 {
1334 u64GuestCR0 |= X86_CR0_PG; /* When Nested Paging is not available, use shadow page tables. */
1335 u64GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF #VMEXIT. */
1336 }
1337
1338 /*
1339 * Guest FPU bits.
1340 */
1341 bool fInterceptNM = false;
1342 bool fInterceptMF = false;
1343 u64GuestCR0 |= X86_CR0_NE; /* Use internal x87 FPU exceptions handling rather than external interrupts. */
1344 if (CPUMIsGuestFPUStateActive(pVCpu))
1345 {
1346 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
1347 if (!(pCtx->cr0 & X86_CR0_NE))
1348 {
1349 Log4(("hmR0SvmLoadSharedCR0: Intercepting Guest CR0.MP Old-style FPU handling!!!\n"));
1350 fInterceptMF = true;
1351 }
1352 }
1353 else
1354 {
1355 fInterceptNM = true; /* Guest FPU inactive, #VMEXIT on #NM for lazy FPU loading. */
1356 u64GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
1357 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
1358 }
1359
1360 /*
1361 * Update the exception intercept bitmap.
1362 */
1363 if (fInterceptNM)
1364 hmR0SvmAddXcptIntercept(pVmcb, X86_XCPT_NM);
1365 else
1366 hmR0SvmRemoveXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_NM);
1367
1368 if (fInterceptMF)
1369 hmR0SvmAddXcptIntercept(pVmcb, X86_XCPT_MF);
1370 else
1371 hmR0SvmRemoveXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_MF);
1372
1373 pVmcb->guest.u64CR0 = u64GuestCR0;
1374 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1375}
1376
1377
1378/**
1379 * Loads the guest/nested-guest control registers (CR2, CR3, CR4) into the VMCB.
1380 *
1381 * @returns VBox status code.
1382 * @param pVCpu The cross context virtual CPU structure.
1383 * @param pVmcb Pointer to the VM control block.
1384 * @param pCtx Pointer to the guest-CPU context.
1385 *
1386 * @remarks No-long-jump zone!!!
1387 */
1388static int hmR0SvmLoadGuestControlRegs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1389{
1390 PVM pVM = pVCpu->CTX_SUFF(pVM);
1391
1392 /*
1393 * Guest CR2.
1394 */
1395 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR2))
1396 {
1397 pVmcb->guest.u64CR2 = pCtx->cr2;
1398 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CR2;
1399 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
1400 }
1401
1402 /*
1403 * Guest CR3.
1404 */
1405 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
1406 {
1407 if (pVM->hm.s.fNestedPaging)
1408 {
1409 PGMMODE enmShwPagingMode;
1410#if HC_ARCH_BITS == 32
1411 if (CPUMIsGuestInLongModeEx(pCtx))
1412 enmShwPagingMode = PGMMODE_AMD64_NX;
1413 else
1414#endif
1415 enmShwPagingMode = PGMGetHostMode(pVM);
1416
1417 pVmcb->ctrl.u64NestedPagingCR3 = PGMGetNestedCR3(pVCpu, enmShwPagingMode);
1418 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_NP;
1419 Assert(pVmcb->ctrl.u64NestedPagingCR3);
1420 pVmcb->guest.u64CR3 = pCtx->cr3;
1421 }
1422 else
1423 {
1424 pVmcb->guest.u64CR3 = PGMGetHyperCR3(pVCpu);
1425 Log4(("hmR0SvmLoadGuestControlRegs: CR3=%#RX64 (HyperCR3=%#RX64)\n", pCtx->cr3, pVmcb->guest.u64CR3));
1426 }
1427
1428 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1429 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
1430 }
1431
1432 /*
1433 * Guest CR4.
1434 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
1435 */
1436 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
1437 {
1438 uint64_t u64GuestCR4 = pCtx->cr4;
1439 Assert(RT_HI_U32(u64GuestCR4) == 0);
1440 if (!pVM->hm.s.fNestedPaging)
1441 {
1442 switch (pVCpu->hm.s.enmShadowMode)
1443 {
1444 case PGMMODE_REAL:
1445 case PGMMODE_PROTECTED: /* Protected mode, no paging. */
1446 AssertFailed();
1447 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1448
1449 case PGMMODE_32_BIT: /* 32-bit paging. */
1450 u64GuestCR4 &= ~X86_CR4_PAE;
1451 break;
1452
1453 case PGMMODE_PAE: /* PAE paging. */
1454 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1455 /** Must use PAE paging as we could use physical memory > 4 GB */
1456 u64GuestCR4 |= X86_CR4_PAE;
1457 break;
1458
1459 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1460 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1461#ifdef VBOX_ENABLE_64_BITS_GUESTS
1462 break;
1463#else
1464 AssertFailed();
1465 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1466#endif
1467
1468 default: /* shut up gcc */
1469 AssertFailed();
1470 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1471 }
1472 }
1473
1474 pVmcb->guest.u64CR4 = u64GuestCR4;
1475 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1476
1477 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
1478 pVCpu->hm.s.fLoadSaveGuestXcr0 = (u64GuestCR4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
1479
1480 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
1481 }
1482
1483 return VINF_SUCCESS;
1484}
1485
1486
1487/**
1488 * Loads the guest segment registers into the VMCB.
1489 *
1490 * @returns VBox status code.
1491 * @param pVCpu The cross context virtual CPU structure.
1492 * @param pVmcb Pointer to the VM control block.
1493 * @param pCtx Pointer to the guest-CPU context.
1494 *
1495 * @remarks No-long-jump zone!!!
1496 */
1497static void hmR0SvmLoadGuestSegmentRegs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1498{
1499 /* Guest Segment registers: CS, SS, DS, ES, FS, GS. */
1500 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
1501 {
1502 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, CS, cs);
1503 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, SS, ss);
1504 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, DS, ds);
1505 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, ES, es);
1506 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, FS, fs);
1507 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, GS, gs);
1508
1509 pVmcb->guest.u8CPL = pCtx->ss.Attr.n.u2Dpl;
1510 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_SEG;
1511 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
1512 }
1513
1514 /* Guest TR. */
1515 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
1516 {
1517 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, TR, tr);
1518 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
1519 }
1520
1521 /* Guest LDTR. */
1522 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
1523 {
1524 HMSVM_SEG_REG_COPY_TO_VMCB(pCtx, &pVmcb->guest, LDTR, ldtr);
1525 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
1526 }
1527
1528 /* Guest GDTR. */
1529 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
1530 {
1531 pVmcb->guest.GDTR.u32Limit = pCtx->gdtr.cbGdt;
1532 pVmcb->guest.GDTR.u64Base = pCtx->gdtr.pGdt;
1533 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1534 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
1535 }
1536
1537 /* Guest IDTR. */
1538 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
1539 {
1540 pVmcb->guest.IDTR.u32Limit = pCtx->idtr.cbIdt;
1541 pVmcb->guest.IDTR.u64Base = pCtx->idtr.pIdt;
1542 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DT;
1543 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
1544 }
1545}
1546
1547
1548/**
1549 * Loads the guest MSRs into the VMCB.
1550 *
1551 * @param pVCpu The cross context virtual CPU structure.
1552 * @param pVmcb Pointer to the VM control block.
1553 * @param pCtx Pointer to the guest-CPU context.
1554 *
1555 * @remarks No-long-jump zone!!!
1556 */
1557static void hmR0SvmLoadGuestMsrs(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1558{
1559 /* Guest Sysenter MSRs. */
1560 pVmcb->guest.u64SysEnterCS = pCtx->SysEnter.cs;
1561 pVmcb->guest.u64SysEnterEIP = pCtx->SysEnter.eip;
1562 pVmcb->guest.u64SysEnterESP = pCtx->SysEnter.esp;
1563
1564 /*
1565 * Guest EFER MSR.
1566 * AMD-V requires guest EFER.SVME to be set. Weird.
1567 * See AMD spec. 15.5.1 "Basic Operation" | "Canonicalization and Consistency Checks".
1568 */
1569 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
1570 {
1571 pVmcb->guest.u64EFER = pCtx->msrEFER | MSR_K6_EFER_SVME;
1572 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1573 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
1574 }
1575
1576 /* 64-bit MSRs. */
1577 if (CPUMIsGuestInLongModeEx(pCtx))
1578 {
1579 pVmcb->guest.FS.u64Base = pCtx->fs.u64Base;
1580 pVmcb->guest.GS.u64Base = pCtx->gs.u64Base;
1581 }
1582 else
1583 {
1584 /* If the guest isn't in 64-bit mode, clear MSR_K6_LME bit from guest EFER otherwise AMD-V expects amd64 shadow paging. */
1585 if (pCtx->msrEFER & MSR_K6_EFER_LME)
1586 {
1587 pVmcb->guest.u64EFER &= ~MSR_K6_EFER_LME;
1588 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_CRX_EFER;
1589 }
1590 }
1591
1592 /** @todo The following are used in 64-bit only (SYSCALL/SYSRET) but they might
1593 * be writable in 32-bit mode. Clarify with AMD spec. */
1594 pVmcb->guest.u64STAR = pCtx->msrSTAR;
1595 pVmcb->guest.u64LSTAR = pCtx->msrLSTAR;
1596 pVmcb->guest.u64CSTAR = pCtx->msrCSTAR;
1597 pVmcb->guest.u64SFMASK = pCtx->msrSFMASK;
1598 pVmcb->guest.u64KernelGSBase = pCtx->msrKERNELGSBASE;
1599}
1600
1601
1602/**
1603 * Loads the guest (or nested-guest) debug state into the VMCB and programs the
1604 * necessary intercepts accordingly.
1605 *
1606 * @param pVCpu The cross context virtual CPU structure.
1607 * @param pVmcb Pointer to the VM control block.
1608 * @param pCtx Pointer to the guest-CPU context.
1609 *
1610 * @remarks No-long-jump zone!!!
1611 * @remarks Requires EFLAGS to be up-to-date in the VMCB!
1612 */
1613static void hmR0SvmLoadSharedDebugState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1614{
1615 bool fInterceptMovDRx = false;
1616
1617 /*
1618 * Anyone single stepping on the host side? If so, we'll have to use the
1619 * trap flag in the guest EFLAGS since AMD-V doesn't have a trap flag on
1620 * the VMM level like the VT-x implementations does.
1621 */
1622 bool const fStepping = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
1623 if (fStepping)
1624 {
1625 pVCpu->hm.s.fClearTrapFlag = true;
1626 pVmcb->guest.u64RFlags |= X86_EFL_TF;
1627 fInterceptMovDRx = true; /* Need clean DR6, no guest mess. */
1628 }
1629
1630 if ( fStepping
1631 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
1632 {
1633 /*
1634 * Use the combined guest and host DRx values found in the hypervisor
1635 * register set because the debugger has breakpoints active or someone
1636 * is single stepping on the host side.
1637 *
1638 * Note! DBGF expects a clean DR6 state before executing guest code.
1639 */
1640#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1641 if ( CPUMIsGuestInLongModeEx(pCtx)
1642 && !CPUMIsHyperDebugStateActivePending(pVCpu))
1643 {
1644 CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1645 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
1646 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
1647 }
1648 else
1649#endif
1650 if (!CPUMIsHyperDebugStateActive(pVCpu))
1651 {
1652 CPUMR0LoadHyperDebugState(pVCpu, false /* include DR6 */);
1653 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
1654 Assert(CPUMIsHyperDebugStateActive(pVCpu));
1655 }
1656
1657 /* Update DR6 & DR7. (The other DRx values are handled by CPUM one way or the other.) */
1658 if ( pVmcb->guest.u64DR6 != X86_DR6_INIT_VAL
1659 || pVmcb->guest.u64DR7 != CPUMGetHyperDR7(pVCpu))
1660 {
1661 pVmcb->guest.u64DR7 = CPUMGetHyperDR7(pVCpu);
1662 pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
1663 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1664 pVCpu->hm.s.fUsingHyperDR7 = true;
1665 }
1666
1667 /** @todo If we cared, we could optimize to allow the guest to read registers
1668 * with the same values. */
1669 fInterceptMovDRx = true;
1670 Log5(("hmR0SvmLoadSharedDebugState: Loaded hyper DRx\n"));
1671 }
1672 else
1673 {
1674 /*
1675 * Update DR6, DR7 with the guest values if necessary.
1676 */
1677 if ( pVmcb->guest.u64DR7 != pCtx->dr[7]
1678 || pVmcb->guest.u64DR6 != pCtx->dr[6])
1679 {
1680 pVmcb->guest.u64DR7 = pCtx->dr[7];
1681 pVmcb->guest.u64DR6 = pCtx->dr[6];
1682 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
1683 pVCpu->hm.s.fUsingHyperDR7 = false;
1684 }
1685
1686 /*
1687 * If the guest has enabled debug registers, we need to load them prior to
1688 * executing guest code so they'll trigger at the right time.
1689 */
1690 if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
1691 {
1692#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1693 if ( CPUMIsGuestInLongModeEx(pCtx)
1694 && !CPUMIsGuestDebugStateActivePending(pVCpu))
1695 {
1696 CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1697 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1698 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
1699 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
1700 }
1701 else
1702#endif
1703 if (!CPUMIsGuestDebugStateActive(pVCpu))
1704 {
1705 CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
1706 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
1707 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
1708 Assert(CPUMIsGuestDebugStateActive(pVCpu));
1709 }
1710 Log5(("hmR0SvmLoadSharedDebugState: Loaded guest DRx\n"));
1711 }
1712 /*
1713 * If no debugging enabled, we'll lazy load DR0-3. We don't need to
1714 * intercept #DB as DR6 is updated in the VMCB.
1715 *
1716 * Note! If we cared and dared, we could skip intercepting \#DB here.
1717 * However, \#DB shouldn't be performance critical, so we'll play safe
1718 * and keep the code similar to the VT-x code and always intercept it.
1719 */
1720#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
1721 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
1722 && !CPUMIsGuestDebugStateActive(pVCpu))
1723#else
1724 else if (!CPUMIsGuestDebugStateActive(pVCpu))
1725#endif
1726 {
1727 fInterceptMovDRx = true;
1728 }
1729 }
1730
1731 Assert(pVmcb->ctrl.u32InterceptXcpt & RT_BIT_32(X86_XCPT_DB));
1732 if (fInterceptMovDRx)
1733 {
1734 if ( pVmcb->ctrl.u16InterceptRdDRx != 0xffff
1735 || pVmcb->ctrl.u16InterceptWrDRx != 0xffff)
1736 {
1737 pVmcb->ctrl.u16InterceptRdDRx = 0xffff;
1738 pVmcb->ctrl.u16InterceptWrDRx = 0xffff;
1739 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1740 }
1741 }
1742 else
1743 {
1744 if ( pVmcb->ctrl.u16InterceptRdDRx
1745 || pVmcb->ctrl.u16InterceptWrDRx)
1746 {
1747 pVmcb->ctrl.u16InterceptRdDRx = 0;
1748 pVmcb->ctrl.u16InterceptWrDRx = 0;
1749 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1750 }
1751 }
1752 Log4(("hmR0SvmLoadSharedDebugState: DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
1753}
1754
1755
1756#ifdef VBOX_WITH_NESTED_HWVIRT
1757/**
1758 * Loads the nested-guest APIC state (currently just the TPR).
1759 *
1760 * @param pVCpu The cross context virtual CPU structure.
1761 * @param pVmcbNstGst Pointer to the nested-guest VM control block.
1762 */
1763static void hmR0SvmLoadGuestApicStateNested(PVMCPU pVCpu, PSVMVMCB pVmcbNstGst)
1764{
1765 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE))
1766 {
1767 /* Always enable V_INTR_MASKING as we do not want to allow access to the physical APIC TPR. */
1768 pVmcbNstGst->ctrl.IntCtrl.n.u1VIntrMasking = 1;
1769 pVCpu->hm.s.svm.fSyncVTpr = false;
1770 pVmcbNstGst->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_TPR;
1771
1772 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
1773 }
1774}
1775#endif
1776
1777/**
1778 * Loads the guest APIC state (currently just the TPR).
1779 *
1780 * @returns VBox status code.
1781 * @param pVCpu The cross context virtual CPU structure.
1782 * @param pVmcb Pointer to the VM control block.
1783 * @param pCtx Pointer to the guest-CPU context.
1784 */
1785static int hmR0SvmLoadGuestApicState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1786{
1787 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE))
1788 return VINF_SUCCESS;
1789
1790 int rc = VINF_SUCCESS;
1791 PVM pVM = pVCpu->CTX_SUFF(pVM);
1792 if ( PDMHasApic(pVM)
1793 && APICIsEnabled(pVCpu))
1794 {
1795 bool fPendingIntr;
1796 uint8_t u8Tpr;
1797 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, NULL /* pu8PendingIrq */);
1798 AssertRCReturn(rc, rc);
1799
1800 /* Assume that we need to trap all TPR accesses and thus need not check on
1801 every #VMEXIT if we should update the TPR. */
1802 Assert(pVmcb->ctrl.IntCtrl.n.u1VIntrMasking);
1803 pVCpu->hm.s.svm.fSyncVTpr = false;
1804
1805 /* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
1806 if (pVM->hm.s.fTPRPatchingActive)
1807 {
1808 pCtx->msrLSTAR = u8Tpr;
1809 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
1810
1811 /* If there are interrupts pending, intercept LSTAR writes, otherwise don't intercept reads or writes. */
1812 if (fPendingIntr)
1813 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_INTERCEPT_WRITE);
1814 else
1815 {
1816 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_LSTAR, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
1817 pVCpu->hm.s.svm.fSyncVTpr = true;
1818 }
1819 }
1820 else
1821 {
1822 /* Bits 3-0 of the VTPR field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
1823 pVmcb->ctrl.IntCtrl.n.u8VTPR = (u8Tpr >> 4);
1824
1825 /* If there are interrupts pending, intercept CR8 writes to evaluate ASAP if we can deliver the interrupt to the guest. */
1826 if (fPendingIntr)
1827 pVmcb->ctrl.u16InterceptWrCRx |= RT_BIT(8);
1828 else
1829 {
1830 pVmcb->ctrl.u16InterceptWrCRx &= ~RT_BIT(8);
1831 pVCpu->hm.s.svm.fSyncVTpr = true;
1832 }
1833
1834 pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS | HMSVM_VMCB_CLEAN_TPR);
1835 }
1836 }
1837
1838 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
1839 return rc;
1840}
1841
1842
1843/**
1844 * Loads the exception interrupts required for guest (or nested-guest) execution in
1845 * the VMCB.
1846 *
1847 * @param pVCpu The cross context virtual CPU structure.
1848 * @param pVmcb Pointer to the VM control block.
1849 * @param pCtx Pointer to the guest-CPU context.
1850 */
1851static void hmR0SvmLoadGuestXcptIntercepts(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
1852{
1853 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
1854 {
1855 /* Trap #UD for GIM provider (e.g. for hypercalls). */
1856 if (pVCpu->hm.s.fGIMTrapXcptUD)
1857 hmR0SvmAddXcptIntercept(pVmcb, X86_XCPT_UD);
1858 else
1859 hmR0SvmRemoveXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_UD);
1860
1861 /* Trap #BP for INT3 debug breakpoints set by the VM debugger. */
1862 if (pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
1863 hmR0SvmAddXcptIntercept(pVmcb, X86_XCPT_BP);
1864 else
1865 hmR0SvmRemoveXcptIntercept(pVCpu, pCtx, pVmcb, X86_XCPT_BP);
1866
1867 /* The remaining intercepts are handled elsewhere, e.g. in hmR0SvmLoadSharedCR0(). */
1868 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
1869 }
1870}
1871
1872
1873#ifdef VBOX_WITH_NESTED_HWVIRT
1874/**
1875 * Loads the intercepts required for nested-guest execution in the VMCB.
1876 *
1877 * This merges the guest and nested-guest intercepts in a way that if the outer
1878 * guest intercepts an exception we need to intercept it in the nested-guest as
1879 * well and handle it accordingly.
1880 *
1881 * @param pVCpu The cross context virtual CPU structure.
1882 * @param pVmcbNstGst Pointer to the nested-guest VM control block.
1883 * @param pCtx Pointer to the guest-CPU context.
1884 */
1885static void hmR0SvmLoadGuestXcptInterceptsNested(PVMCPU pVCpu, PSVMVMCB pVmcbNstGst, PCPUMCTX pCtx)
1886{
1887 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
1888 {
1889 /* First, load the guest intercepts into the guest VMCB. */
1890 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
1891 Assert(!(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR));
1892 hmR0SvmLoadGuestXcptIntercepts(pVCpu, pVmcb, pCtx);
1893
1894 /* Next, merge the intercepts into the nested-guest VMCB. */
1895 pVmcbNstGst->ctrl.u16InterceptRdCRx |= pVmcb->ctrl.u16InterceptRdCRx;
1896 pVmcbNstGst->ctrl.u16InterceptWrCRx |= pVmcb->ctrl.u16InterceptWrCRx;
1897
1898 /* Always intercept CR0, CR4 reads and writes as we alter them. */
1899 pVmcbNstGst->ctrl.u16InterceptRdCRx |= RT_BIT(0) | RT_BIT(4);
1900 pVmcbNstGst->ctrl.u16InterceptWrCRx |= RT_BIT(0) | RT_BIT(4);
1901
1902 /* Always intercept CR3 reads and writes without nested-paging as we load shadow page tables. */
1903 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
1904 {
1905 pVmcbNstGst->ctrl.u16InterceptRdCRx |= RT_BIT(3);
1906 pVmcbNstGst->ctrl.u16InterceptWrCRx |= RT_BIT(3);
1907 }
1908
1909 /** @todo Figure out debugging with nested-guests, till then just intercept
1910 * all DR[0-15] accesses. */
1911 pVmcbNstGst->ctrl.u16InterceptRdDRx |= 0xffff;
1912 pVmcbNstGst->ctrl.u16InterceptWrDRx |= 0xffff;
1913
1914 pVmcbNstGst->ctrl.u32InterceptXcpt |= pVmcb->ctrl.u32InterceptXcpt;
1915 pVmcbNstGst->ctrl.u64InterceptCtrl |= pVmcb->ctrl.u64InterceptCtrl
1916 | HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS;
1917 /*
1918 * Remove control intercepts that we don't need while executing the nested-guest.
1919 *
1920 * VMMCALL when not intercepted raises a \#UD exception in the guest. However,
1921 * other SVM instructions like VMSAVE when not intercept can cause havoc on the
1922 * host as they can write to any location in physical memory, hence they always
1923 * need to be intercepted (see below).
1924 */
1925 Assert( (pVmcbNstGst->ctrl.u64InterceptCtrl & HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS)
1926 == HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS);
1927 pVmcbNstGst->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_VMMCALL;
1928
1929 /*
1930 * If we don't expose Virtualized-VMSAVE/VMLOAD feature to the outer guest, we
1931 * need to intercept VMSAVE/VMLOAD instructions executed by the nested-guest.
1932 */
1933 if (!pVCpu->CTX_SUFF(pVM)->cpum.ro.GuestFeatures.fSvmVirtVmsaveVmload)
1934 {
1935 pVmcbNstGst->ctrl.u64InterceptCtrl |= SVM_CTRL_INTERCEPT_VMSAVE
1936 | SVM_CTRL_INTERCEPT_VMLOAD;
1937 }
1938
1939 /* Finally, update the VMCB clean bits. */
1940 pVmcbNstGst->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
1941
1942 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS));
1943 }
1944}
1945#endif
1946
1947
1948/**
1949 * Sets up the appropriate function to run guest code.
1950 *
1951 * @returns VBox status code.
1952 * @param pVCpu The cross context virtual CPU structure.
1953 *
1954 * @remarks No-long-jump zone!!!
1955 */
1956static int hmR0SvmSetupVMRunHandler(PVMCPU pVCpu)
1957{
1958 if (CPUMIsGuestInLongMode(pVCpu))
1959 {
1960#ifndef VBOX_ENABLE_64_BITS_GUESTS
1961 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1962#endif
1963 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
1964#if HC_ARCH_BITS == 32
1965 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
1966 pVCpu->hm.s.svm.pfnVMRun = SVMR0VMSwitcherRun64;
1967#else
1968 /* 64-bit host or hybrid host. */
1969 pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun64;
1970#endif
1971 }
1972 else
1973 {
1974 /* Guest is not in long mode, use the 32-bit handler. */
1975 pVCpu->hm.s.svm.pfnVMRun = SVMR0VMRun;
1976 }
1977 return VINF_SUCCESS;
1978}
1979
1980
1981/**
1982 * Enters the AMD-V session.
1983 *
1984 * @returns VBox status code.
1985 * @param pVM The cross context VM structure.
1986 * @param pVCpu The cross context virtual CPU structure.
1987 * @param pCpu Pointer to the CPU info struct.
1988 */
1989VMMR0DECL(int) SVMR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1990{
1991 AssertPtr(pVM);
1992 AssertPtr(pVCpu);
1993 Assert(pVM->hm.s.svm.fSupported);
1994 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1995 NOREF(pVM); NOREF(pCpu);
1996
1997 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1998 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
1999
2000 pVCpu->hm.s.fLeaveDone = false;
2001 return VINF_SUCCESS;
2002}
2003
2004
2005/**
2006 * Thread-context callback for AMD-V.
2007 *
2008 * @param enmEvent The thread-context event.
2009 * @param pVCpu The cross context virtual CPU structure.
2010 * @param fGlobalInit Whether global VT-x/AMD-V init. is used.
2011 * @thread EMT(pVCpu)
2012 */
2013VMMR0DECL(void) SVMR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
2014{
2015 NOREF(fGlobalInit);
2016
2017 switch (enmEvent)
2018 {
2019 case RTTHREADCTXEVENT_OUT:
2020 {
2021 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2022 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
2023 VMCPU_ASSERT_EMT(pVCpu);
2024
2025 /* No longjmps (log-flush, locks) in this fragile context. */
2026 VMMRZCallRing3Disable(pVCpu);
2027
2028 if (!pVCpu->hm.s.fLeaveDone)
2029 {
2030 hmR0SvmLeave(pVCpu);
2031 pVCpu->hm.s.fLeaveDone = true;
2032 }
2033
2034 /* Leave HM context, takes care of local init (term). */
2035 int rc = HMR0LeaveCpu(pVCpu);
2036 AssertRC(rc); NOREF(rc);
2037
2038 /* Restore longjmp state. */
2039 VMMRZCallRing3Enable(pVCpu);
2040 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
2041 break;
2042 }
2043
2044 case RTTHREADCTXEVENT_IN:
2045 {
2046 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2047 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
2048 VMCPU_ASSERT_EMT(pVCpu);
2049
2050 /* No longjmps (log-flush, locks) in this fragile context. */
2051 VMMRZCallRing3Disable(pVCpu);
2052
2053 /*
2054 * Initialize the bare minimum state required for HM. This takes care of
2055 * initializing AMD-V if necessary (onlined CPUs, local init etc.)
2056 */
2057 int rc = HMR0EnterCpu(pVCpu);
2058 AssertRC(rc); NOREF(rc);
2059 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
2060
2061 pVCpu->hm.s.fLeaveDone = false;
2062
2063 /* Restore longjmp state. */
2064 VMMRZCallRing3Enable(pVCpu);
2065 break;
2066 }
2067
2068 default:
2069 break;
2070 }
2071}
2072
2073
2074/**
2075 * Saves the host state.
2076 *
2077 * @returns VBox status code.
2078 * @param pVM The cross context VM structure.
2079 * @param pVCpu The cross context virtual CPU structure.
2080 *
2081 * @remarks No-long-jump zone!!!
2082 */
2083VMMR0DECL(int) SVMR0SaveHostState(PVM pVM, PVMCPU pVCpu)
2084{
2085 NOREF(pVM);
2086 NOREF(pVCpu);
2087 /* Nothing to do here. AMD-V does this for us automatically during the world-switch. */
2088 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
2089 return VINF_SUCCESS;
2090}
2091
2092
2093/**
2094 * Loads the guest state into the VMCB.
2095 *
2096 * The CPU state will be loaded from these fields on every successful VM-entry.
2097 * Also sets up the appropriate VMRUN function to execute guest code based on
2098 * the guest CPU mode.
2099 *
2100 * @returns VBox status code.
2101 * @param pVM The cross context VM structure.
2102 * @param pVCpu The cross context virtual CPU structure.
2103 * @param pCtx Pointer to the guest-CPU context.
2104 *
2105 * @remarks No-long-jump zone!!!
2106 */
2107static int hmR0SvmLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2108{
2109 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
2110
2111 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
2112 AssertMsgReturn(pVmcb, ("Invalid pVmcb\n"), VERR_SVM_INVALID_PVMCB);
2113
2114 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
2115
2116 int rc = hmR0SvmLoadGuestControlRegs(pVCpu, pVmcb, pCtx);
2117 AssertLogRelMsgRCReturn(rc, ("hmR0SvmLoadGuestControlRegs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
2118
2119 hmR0SvmLoadGuestSegmentRegs(pVCpu, pVmcb, pCtx);
2120 hmR0SvmLoadGuestMsrs(pVCpu, pVmcb, pCtx);
2121
2122 pVmcb->guest.u64RIP = pCtx->rip;
2123 pVmcb->guest.u64RSP = pCtx->rsp;
2124 pVmcb->guest.u64RFlags = pCtx->eflags.u32;
2125 pVmcb->guest.u64RAX = pCtx->rax;
2126
2127 rc = hmR0SvmLoadGuestApicState(pVCpu, pVmcb, pCtx);
2128 AssertLogRelMsgRCReturn(rc, ("hmR0SvmLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
2129
2130 hmR0SvmLoadGuestXcptIntercepts(pVCpu, pVmcb, pCtx);
2131
2132 rc = hmR0SvmSetupVMRunHandler(pVCpu);
2133 AssertLogRelMsgRCReturn(rc, ("hmR0SvmSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
2134
2135 /* Clear any unused and reserved bits. */
2136 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP /* Unused (loaded unconditionally). */
2137 | HM_CHANGED_GUEST_RSP
2138 | HM_CHANGED_GUEST_RFLAGS
2139 | HM_CHANGED_GUEST_SYSENTER_CS_MSR
2140 | HM_CHANGED_GUEST_SYSENTER_EIP_MSR
2141 | HM_CHANGED_GUEST_SYSENTER_ESP_MSR
2142 | HM_CHANGED_GUEST_LAZY_MSRS /* Unused. */
2143 | HM_CHANGED_SVM_RESERVED1 /* Reserved. */
2144 | HM_CHANGED_SVM_RESERVED2
2145 | HM_CHANGED_SVM_RESERVED3
2146 | HM_CHANGED_SVM_RESERVED4);
2147
2148 /* All the guest state bits should be loaded except maybe the host context and/or shared host/guest bits. */
2149 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
2150 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
2151 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
2152
2153 Log4(("hmR0SvmLoadGuestState: CS:RIP=%04x:%RX64 EFL=%#x CR0=%#RX32 CR3=%#RX32 CR4=%#RX32 ESP=%#RX32 EBP=%#RX32\n",
2154 pCtx->cs.Sel, pCtx->rip, pCtx->eflags.u, pCtx->cr0, pCtx->cr3, pCtx->cr4, pCtx->esp, pCtx->ebp));
2155 Log4(("hmR0SvmLoadGuestState: SS={%04x base=%016RX64 limit=%08x flags=%08x}\n", pCtx->ss.Sel, pCtx->ss.u64Base,
2156 pCtx->ss.u32Limit, pCtx->ss.Attr.u));
2157 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
2158 return rc;
2159}
2160
2161
2162#ifdef VBOX_WITH_NESTED_HWVIRT
2163/**
2164 * Caches the nested-guest VMCB fields before we modify them for execution using
2165 * hardware-assisted SVM.
2166 *
2167 * @returns true if the VMCB was previously already cached, false otherwise.
2168 * @param pCtx Pointer to the guest-CPU context.
2169 *
2170 * @sa HMSvmNstGstVmExitNotify.
2171 */
2172static bool hmR0SvmVmRunCacheVmcb(PVMCPU pVCpu, PCPUMCTX pCtx)
2173{
2174 PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2175 PCSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2176 PCSVMVMCBSTATESAVE pVmcbNstGstState = &pVmcbNstGst->guest;
2177 PSVMNESTEDVMCBCACHE pNstGstVmcbCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2178
2179 /*
2180 * Cache the nested-guest programmed VMCB fields if we have not cached it yet.
2181 * Otherwise we risk re-caching the values we may have modified, see @bugref{7243#c44}.
2182 *
2183 * Nested-paging CR3 is not saved back into the VMCB on #VMEXIT, hence no need to
2184 * cache and restore it, see AMD spec. 15.25.4 "Nested Paging and VMRUN/#VMEXIT".
2185 */
2186 bool const fWasCached = pCtx->hwvirt.svm.fHMCachedVmcb;
2187 if (!fWasCached)
2188 {
2189 pNstGstVmcbCache->u16InterceptRdCRx = pVmcbNstGstCtrl->u16InterceptRdCRx;
2190 pNstGstVmcbCache->u16InterceptWrCRx = pVmcbNstGstCtrl->u16InterceptWrCRx;
2191 pNstGstVmcbCache->u16InterceptRdDRx = pVmcbNstGstCtrl->u16InterceptRdDRx;
2192 pNstGstVmcbCache->u16InterceptWrDRx = pVmcbNstGstCtrl->u16InterceptWrDRx;
2193 pNstGstVmcbCache->u32InterceptXcpt = pVmcbNstGstCtrl->u32InterceptXcpt;
2194 pNstGstVmcbCache->u64InterceptCtrl = pVmcbNstGstCtrl->u64InterceptCtrl;
2195 pNstGstVmcbCache->u64CR0 = pVmcbNstGstState->u64CR0;
2196 pNstGstVmcbCache->u64CR3 = pVmcbNstGstState->u64CR3;
2197 pNstGstVmcbCache->u64CR4 = pVmcbNstGstState->u64CR4;
2198 pNstGstVmcbCache->u64EFER = pVmcbNstGstState->u64EFER;
2199 pNstGstVmcbCache->u64DBGCTL = pVmcbNstGstState->u64DBGCTL;
2200 pNstGstVmcbCache->u64IOPMPhysAddr = pVmcbNstGstCtrl->u64IOPMPhysAddr;
2201 pNstGstVmcbCache->u64MSRPMPhysAddr = pVmcbNstGstCtrl->u64MSRPMPhysAddr;
2202 pNstGstVmcbCache->u64TSCOffset = pVmcbNstGstCtrl->u64TSCOffset;
2203 pNstGstVmcbCache->u32VmcbCleanBits = pVmcbNstGstCtrl->u32VmcbCleanBits;
2204 pNstGstVmcbCache->fVIntrMasking = pVmcbNstGstCtrl->IntCtrl.n.u1VIntrMasking;
2205 pNstGstVmcbCache->TLBCtrl = pVmcbNstGstCtrl->TLBCtrl;
2206 pNstGstVmcbCache->u1NestedPaging = pVmcbNstGstCtrl->NestedPaging.n.u1NestedPaging;
2207 pNstGstVmcbCache->u1LbrVirt = pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt;
2208 pCtx->hwvirt.svm.fHMCachedVmcb = true;
2209 Log4(("hmR0SvmVmRunCacheVmcb: Cached VMCB fields\n"));
2210 }
2211
2212 return fWasCached;
2213}
2214
2215
2216/**
2217 * Sets up the nested-guest VMCB for execution using hardware-assisted SVM.
2218 *
2219 * @param pVCpu The cross context virtual CPU structure.
2220 * @param pCtx Pointer to the guest-CPU context.
2221 */
2222static void hmR0SvmVmRunSetupVmcb(PVMCPU pVCpu, PCPUMCTX pCtx)
2223{
2224 PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2225 PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2226
2227 /*
2228 * First cache the nested-guest VMCB fields we may potentially modify.
2229 */
2230 bool const fVmcbCached = hmR0SvmVmRunCacheVmcb(pVCpu, pCtx);
2231 if (!fVmcbCached)
2232 {
2233 /*
2234 * The IOPM of the nested-guest can be ignored because the the guest always
2235 * intercepts all IO port accesses. Thus, we'll swap to the guest IOPM rather
2236 * into the nested-guest one and swap it back on the #VMEXIT.
2237 */
2238 pVmcbNstGstCtrl->u64IOPMPhysAddr = g_HCPhysIOBitmap;
2239
2240 /*
2241 * Load the host-physical address into the MSRPM rather than the nested-guest
2242 * physical address (currently we trap all MSRs in the nested-guest).
2243 */
2244 pVmcbNstGstCtrl->u64MSRPMPhysAddr = g_HCPhysNstGstMsrBitmap;
2245
2246 /*
2247 * Use the same nested-paging as the "outer" guest. We can't dynamically
2248 * switch off nested-paging suddenly while executing a VM (see assertion at the
2249 * end of Trap0eHandler in PGMAllBth.h).
2250 */
2251 pVmcbNstGstCtrl->NestedPaging.n.u1NestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
2252
2253 /* For now copy the LBR info. from outer guest VMCB. */
2254 /** @todo fix this later. */
2255 PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
2256 pVmcbNstGstCtrl->LbrVirt.n.u1LbrVirt = pVmcb->ctrl.LbrVirt.n.u1LbrVirt;
2257 pVmcbNstGst->guest.u64DBGCTL = pVmcb->guest.u64DBGCTL;
2258 }
2259 else
2260 {
2261 Assert(pVmcbNstGstCtrl->u64IOPMPhysAddr == g_HCPhysIOBitmap);
2262 Assert(pVmcbNstGstCtrl->u64MSRPMPhysAddr = g_HCPhysNstGstMsrBitmap);
2263 Assert(RT_BOOL(pVmcbNstGstCtrl->NestedPaging.n.u1NestedPaging) == pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
2264 }
2265}
2266
2267
2268/**
2269 * Loads the nested-guest state into the VMCB.
2270 *
2271 * @returns VBox status code.
2272 * @param pVCpu The cross context virtual CPU structure.
2273 * @param pCtx Pointer to the guest-CPU context.
2274 *
2275 * @remarks No-long-jump zone!!!
2276 */
2277static int hmR0SvmLoadGuestStateNested(PVMCPU pVCpu, PCPUMCTX pCtx)
2278{
2279 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
2280
2281 PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
2282 Assert(pVmcbNstGst);
2283
2284 hmR0SvmVmRunSetupVmcb(pVCpu, pCtx);
2285
2286 int rc = hmR0SvmLoadGuestControlRegs(pVCpu, pVmcbNstGst, pCtx);
2287 AssertRCReturn(rc, rc);
2288
2289 hmR0SvmLoadGuestSegmentRegs(pVCpu, pVmcbNstGst, pCtx);
2290 hmR0SvmLoadGuestMsrs(pVCpu, pVmcbNstGst, pCtx);
2291 hmR0SvmLoadGuestApicStateNested(pVCpu, pVmcbNstGst);
2292
2293 pVmcbNstGst->guest.u64RIP = pCtx->rip;
2294 pVmcbNstGst->guest.u64RSP = pCtx->rsp;
2295 pVmcbNstGst->guest.u64RFlags = pCtx->eflags.u32;
2296 pVmcbNstGst->guest.u64RAX = pCtx->rax;
2297
2298 hmR0SvmLoadGuestXcptInterceptsNested(pVCpu, pVmcbNstGst, pCtx);
2299
2300 rc = hmR0SvmSetupVMRunHandler(pVCpu);
2301 AssertRCReturn(rc, rc);
2302
2303 /* Clear any unused and reserved bits. */
2304 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP /* Unused (loaded unconditionally). */
2305 | HM_CHANGED_GUEST_RSP
2306 | HM_CHANGED_GUEST_RFLAGS
2307 | HM_CHANGED_GUEST_SYSENTER_CS_MSR
2308 | HM_CHANGED_GUEST_SYSENTER_EIP_MSR
2309 | HM_CHANGED_GUEST_SYSENTER_ESP_MSR
2310 | HM_CHANGED_GUEST_LAZY_MSRS /* Unused. */
2311 | HM_CHANGED_SVM_RESERVED1 /* Reserved. */
2312 | HM_CHANGED_SVM_RESERVED2
2313 | HM_CHANGED_SVM_RESERVED3
2314 | HM_CHANGED_SVM_RESERVED4);
2315
2316 /* All the guest state bits should be loaded except maybe the host context and/or shared host/guest bits. */
2317 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
2318 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
2319 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
2320
2321 Log4(("hmR0SvmLoadGuestStateNested: CS:RIP=%04x:%RX64 EFL=%#x CR0=%#RX32 CR3=%#RX32 (HyperCR3=%#RX64) CR4=%#RX32 "
2322 "ESP=%#RX32 EBP=%#RX32 rc=%d\n", pCtx->cs.Sel, pCtx->rip, pCtx->eflags.u, pCtx->cr0, pCtx->cr3,
2323 pVmcbNstGst->guest.u64CR3, pCtx->cr4, pCtx->esp, pCtx->ebp, rc));
2324 Log4(("hmR0SvmLoadGuestStateNested: SS={%04x base=%016RX64 limit=%08x flags=%08x}\n", pCtx->ss.Sel, pCtx->ss.u64Base,
2325 pCtx->ss.u32Limit, pCtx->ss.Attr.u));
2326 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
2327
2328 return rc;
2329}
2330#endif
2331
2332
2333/**
2334 * Loads the state shared between the host and guest or nested-guest into the
2335 * VMCB.
2336 *
2337 * @param pVCpu The cross context virtual CPU structure.
2338 * @param pVmcb Pointer to the VM control block.
2339 * @param pCtx Pointer to the guest-CPU context.
2340 *
2341 * @remarks No-long-jump zone!!!
2342 */
2343static void hmR0SvmLoadSharedState(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx)
2344{
2345 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2346 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2347
2348 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
2349 {
2350 hmR0SvmLoadSharedCR0(pVCpu, pVmcb, pCtx);
2351 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
2352 }
2353
2354 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
2355 {
2356 /** @todo Figure out stepping with nested-guest. */
2357 if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
2358 hmR0SvmLoadSharedDebugState(pVCpu, pVmcb, pCtx);
2359 else
2360 {
2361 pVmcb->guest.u64DR6 = pCtx->dr[6];
2362 pVmcb->guest.u64DR7 = pCtx->dr[7];
2363 Log4(("hmR0SvmLoadSharedState: DR6=%#RX64 DR7=%#RX64\n", pCtx->dr[6], pCtx->dr[7]));
2364 }
2365
2366 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
2367 }
2368
2369 /* Unused on AMD-V. */
2370 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
2371
2372 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
2373 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
2374}
2375
2376
2377/**
2378 * Saves the guest (or nested-guest) state from the VMCB into the guest-CPU context.
2379 *
2380 * Currently there is no residual state left in the CPU that is not updated in the
2381 * VMCB.
2382 *
2383 * @returns VBox status code.
2384 * @param pVCpu The cross context virtual CPU structure.
2385 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2386 * out-of-sync. Make sure to update the required fields
2387 * before using them.
2388 * @param pVmcb Pointer to the VM control block.
2389 */
2390static void hmR0SvmSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PCSVMVMCB pVmcb)
2391{
2392 Assert(VMMRZCallRing3IsEnabled(pVCpu));
2393
2394 pMixedCtx->rip = pVmcb->guest.u64RIP;
2395 pMixedCtx->rsp = pVmcb->guest.u64RSP;
2396 pMixedCtx->eflags.u32 = pVmcb->guest.u64RFlags;
2397 pMixedCtx->rax = pVmcb->guest.u64RAX;
2398
2399 /*
2400 * Guest interrupt shadow.
2401 */
2402 if (pVmcb->ctrl.IntShadow.n.u1IntShadow)
2403 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
2404 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2405 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2406
2407 /*
2408 * Guest Control registers: CR2, CR3 (handled at the end) - accesses to other control registers are always intercepted.
2409 */
2410 pMixedCtx->cr2 = pVmcb->guest.u64CR2;
2411
2412 /*
2413 * Guest MSRs.
2414 */
2415 pMixedCtx->msrSTAR = pVmcb->guest.u64STAR; /* legacy syscall eip, cs & ss */
2416 pMixedCtx->msrLSTAR = pVmcb->guest.u64LSTAR; /* 64-bit mode syscall rip */
2417 pMixedCtx->msrCSTAR = pVmcb->guest.u64CSTAR; /* compatibility mode syscall rip */
2418 pMixedCtx->msrSFMASK = pVmcb->guest.u64SFMASK; /* syscall flag mask */
2419 pMixedCtx->msrKERNELGSBASE = pVmcb->guest.u64KernelGSBase; /* swapgs exchange value */
2420 pMixedCtx->SysEnter.cs = pVmcb->guest.u64SysEnterCS;
2421 pMixedCtx->SysEnter.eip = pVmcb->guest.u64SysEnterEIP;
2422 pMixedCtx->SysEnter.esp = pVmcb->guest.u64SysEnterESP;
2423
2424 /*
2425 * Guest segment registers (includes FS, GS base MSRs for 64-bit guests).
2426 */
2427 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, CS, cs);
2428 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, SS, ss);
2429 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, DS, ds);
2430 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, ES, es);
2431 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, FS, fs);
2432 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, GS, gs);
2433
2434 /*
2435 * Correct the hidden CS granularity bit. Haven't seen it being wrong in any other
2436 * register (yet).
2437 */
2438 /** @todo SELM might need to be fixed as it too should not care about the
2439 * granularity bit. See @bugref{6785}. */
2440 if ( !pMixedCtx->cs.Attr.n.u1Granularity
2441 && pMixedCtx->cs.Attr.n.u1Present
2442 && pMixedCtx->cs.u32Limit > UINT32_C(0xfffff))
2443 {
2444 Assert((pMixedCtx->cs.u32Limit & 0xfff) == 0xfff);
2445 pMixedCtx->cs.Attr.n.u1Granularity = 1;
2446 }
2447
2448#ifdef VBOX_STRICT
2449# define HMSVM_ASSERT_SEG_GRANULARITY(reg) \
2450 AssertMsg( !pMixedCtx->reg.Attr.n.u1Present \
2451 || ( pMixedCtx->reg.Attr.n.u1Granularity \
2452 ? (pMixedCtx->reg.u32Limit & 0xfff) == 0xfff \
2453 : pMixedCtx->reg.u32Limit <= UINT32_C(0xfffff)), \
2454 ("Invalid Segment Attributes Limit=%#RX32 Attr=%#RX32 Base=%#RX64\n", pMixedCtx->reg.u32Limit, \
2455 pMixedCtx->reg.Attr.u, pMixedCtx->reg.u64Base))
2456
2457 HMSVM_ASSERT_SEG_GRANULARITY(cs);
2458 HMSVM_ASSERT_SEG_GRANULARITY(ss);
2459 HMSVM_ASSERT_SEG_GRANULARITY(ds);
2460 HMSVM_ASSERT_SEG_GRANULARITY(es);
2461 HMSVM_ASSERT_SEG_GRANULARITY(fs);
2462 HMSVM_ASSERT_SEG_GRANULARITY(gs);
2463
2464# undef HMSVM_ASSERT_SEL_GRANULARITY
2465#endif
2466
2467 /*
2468 * Sync the hidden SS DPL field. AMD CPUs have a separate CPL field in the VMCB and uses that
2469 * and thus it's possible that when the CPL changes during guest execution that the SS DPL
2470 * isn't updated by AMD-V. Observed on some AMD Fusion CPUs with 64-bit guests.
2471 * See AMD spec. 15.5.1 "Basic operation".
2472 */
2473 Assert(!(pVmcb->guest.u8CPL & ~0x3));
2474 uint8_t const uCpl = pVmcb->guest.u8CPL;
2475 if (pMixedCtx->ss.Attr.n.u2Dpl != uCpl)
2476 {
2477 Log4(("hmR0SvmSaveGuestState: CPL differs. SS.DPL=%u, CPL=%u, overwriting SS.DPL!\n", pMixedCtx->ss.Attr.n.u2Dpl, uCpl));
2478 pMixedCtx->ss.Attr.n.u2Dpl = pVmcb->guest.u8CPL & 0x3;
2479 }
2480
2481 /*
2482 * Guest TR.
2483 * Fixup TR attributes so it's compatible with Intel. Important when saved-states are used
2484 * between Intel and AMD. See @bugref{6208#c39}.
2485 * ASSUME that it's normally correct and that we're in 32-bit or 64-bit mode.
2486 */
2487 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, TR, tr);
2488 if (pMixedCtx->tr.Attr.n.u4Type != X86_SEL_TYPE_SYS_386_TSS_BUSY)
2489 {
2490 if ( pMixedCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_386_TSS_AVAIL
2491 || CPUMIsGuestInLongModeEx(pMixedCtx))
2492 pMixedCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
2493 else if (pMixedCtx->tr.Attr.n.u4Type == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
2494 pMixedCtx->tr.Attr.n.u4Type = X86_SEL_TYPE_SYS_286_TSS_BUSY;
2495 }
2496
2497 /*
2498 * Guest Descriptor-Table registers.
2499 */
2500 HMSVM_SEG_REG_COPY_FROM_VMCB(pMixedCtx, &pVmcb->guest, LDTR, ldtr);
2501 pMixedCtx->gdtr.cbGdt = pVmcb->guest.GDTR.u32Limit;
2502 pMixedCtx->gdtr.pGdt = pVmcb->guest.GDTR.u64Base;
2503
2504 pMixedCtx->idtr.cbIdt = pVmcb->guest.IDTR.u32Limit;
2505 pMixedCtx->idtr.pIdt = pVmcb->guest.IDTR.u64Base;
2506
2507 /*
2508 * Guest Debug registers.
2509 */
2510 if (!pVCpu->hm.s.fUsingHyperDR7)
2511 {
2512 pMixedCtx->dr[6] = pVmcb->guest.u64DR6;
2513 pMixedCtx->dr[7] = pVmcb->guest.u64DR7;
2514 }
2515 else
2516 {
2517 Assert(pVmcb->guest.u64DR7 == CPUMGetHyperDR7(pVCpu));
2518 CPUMSetHyperDR6(pVCpu, pVmcb->guest.u64DR6);
2519 }
2520
2521 /*
2522 * With Nested Paging, CR3 changes are not intercepted. Therefore, sync. it now.
2523 * This is done as the very last step of syncing the guest state, as PGMUpdateCR3() may cause longjmp's to ring-3.
2524 */
2525 if ( pVmcb->ctrl.NestedPaging.n.u1NestedPaging
2526 && pMixedCtx->cr3 != pVmcb->guest.u64CR3)
2527 {
2528 CPUMSetGuestCR3(pVCpu, pVmcb->guest.u64CR3);
2529 PGMUpdateCR3(pVCpu, pVmcb->guest.u64CR3);
2530 }
2531
2532 if (CPUMIsGuestInSvmNestedHwVirtMode(pMixedCtx))
2533 {
2534 Log4(("hmR0SvmSaveGuestState: CS:RIP=%04x:%RX64 EFL=%#x CR0=%#RX32 CR3=%#RX32 CR4=%#RX32 ESP=%#RX32 EBP=%#RX32\n",
2535 pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->eflags.u, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
2536 pMixedCtx->esp, pMixedCtx->ebp));
2537 Log4(("hmR0SvmSaveGuestState: SS={%04x base=%016RX64 limit=%08x flags=%08x}\n", pMixedCtx->ss.Sel, pMixedCtx->ss.u64Base,
2538 pMixedCtx->ss.u32Limit, pMixedCtx->ss.Attr.u));
2539 Log4(("hmR0SvmSaveGuestState: DBGCTL BR_FROM=%#RX64 BR_TO=%#RX64 XcptFrom=%#RX64 XcptTo=%#RX64\n",
2540 pVmcb->guest.u64BR_FROM, pVmcb->guest.u64BR_TO,pVmcb->guest.u64LASTEXCPFROM, pVmcb->guest.u64LASTEXCPTO));
2541 }
2542}
2543
2544
2545/**
2546 * Does the necessary state syncing before returning to ring-3 for any reason
2547 * (longjmp, preemption, voluntary exits to ring-3) from AMD-V.
2548 *
2549 * @param pVCpu The cross context virtual CPU structure.
2550 *
2551 * @remarks No-long-jmp zone!!!
2552 */
2553static void hmR0SvmLeave(PVMCPU pVCpu)
2554{
2555 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2556 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2557 Assert(VMMR0IsLogFlushDisabled(pVCpu));
2558
2559 /*
2560 * !!! IMPORTANT !!!
2561 * If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
2562 */
2563
2564 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
2565 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
2566 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0); /** @todo r=ramshankar: This shouldn't be necessary, it's set in HMR0EnterCpu. */
2567
2568 /*
2569 * Restore host debug registers if necessary and resync on next R0 reentry.
2570 */
2571#ifdef VBOX_STRICT
2572 if (CPUMIsHyperDebugStateActive(pVCpu))
2573 {
2574 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb; /** @todo nested-guest. */
2575 Assert(pVmcb->ctrl.u16InterceptRdDRx == 0xffff);
2576 Assert(pVmcb->ctrl.u16InterceptWrDRx == 0xffff);
2577 }
2578#endif
2579 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */))
2580 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);/** @todo r=ramshankar: This shouldn't be necessary, it's set in HMR0EnterCpu. */
2581
2582 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
2583 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
2584
2585 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
2586 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
2587 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
2588 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
2589 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
2590
2591 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
2592}
2593
2594
2595/**
2596 * Leaves the AMD-V session.
2597 *
2598 * @returns VBox status code.
2599 * @param pVCpu The cross context virtual CPU structure.
2600 */
2601static int hmR0SvmLeaveSession(PVMCPU pVCpu)
2602{
2603 HM_DISABLE_PREEMPT();
2604 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2605 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2606
2607 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
2608 and done this from the SVMR0ThreadCtxCallback(). */
2609 if (!pVCpu->hm.s.fLeaveDone)
2610 {
2611 hmR0SvmLeave(pVCpu);
2612 pVCpu->hm.s.fLeaveDone = true;
2613 }
2614
2615 /*
2616 * !!! IMPORTANT !!!
2617 * If you modify code here, make sure to check whether hmR0SvmCallRing3Callback() needs to be updated too.
2618 */
2619
2620 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
2621 /* Deregister hook now that we've left HM context before re-enabling preemption. */
2622 VMMR0ThreadCtxHookDisable(pVCpu);
2623
2624 /* Leave HM context. This takes care of local init (term). */
2625 int rc = HMR0LeaveCpu(pVCpu);
2626
2627 HM_RESTORE_PREEMPT();
2628 return rc;
2629}
2630
2631
2632/**
2633 * Does the necessary state syncing before doing a longjmp to ring-3.
2634 *
2635 * @returns VBox status code.
2636 * @param pVCpu The cross context virtual CPU structure.
2637 *
2638 * @remarks No-long-jmp zone!!!
2639 */
2640static int hmR0SvmLongJmpToRing3(PVMCPU pVCpu)
2641{
2642 return hmR0SvmLeaveSession(pVCpu);
2643}
2644
2645
2646/**
2647 * VMMRZCallRing3() callback wrapper which saves the guest state (or restores
2648 * any remaining host state) before we longjump to ring-3 and possibly get
2649 * preempted.
2650 *
2651 * @param pVCpu The cross context virtual CPU structure.
2652 * @param enmOperation The operation causing the ring-3 longjump.
2653 * @param pvUser The user argument (pointer to the possibly
2654 * out-of-date guest-CPU context).
2655 */
2656static DECLCALLBACK(int) hmR0SvmCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
2657{
2658 RT_NOREF_PV(pvUser);
2659
2660 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
2661 {
2662 /*
2663 * !!! IMPORTANT !!!
2664 * If you modify code here, make sure to check whether hmR0SvmLeave() and hmR0SvmLeaveSession() needs
2665 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
2666 */
2667 VMMRZCallRing3RemoveNotification(pVCpu);
2668 VMMRZCallRing3Disable(pVCpu);
2669 HM_DISABLE_PREEMPT();
2670
2671 /* Restore host FPU state if necessary and resync on next R0 reentry. */
2672 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
2673
2674 /* Restore host debug registers if necessary and resync on next R0 reentry. */
2675 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, false /* save DR6 */);
2676
2677 /* Deregister the hook now that we've left HM context before re-enabling preemption. */
2678 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
2679 VMMR0ThreadCtxHookDisable(pVCpu);
2680
2681 /* Leave HM context. This takes care of local init (term). */
2682 HMR0LeaveCpu(pVCpu);
2683
2684 HM_RESTORE_PREEMPT();
2685 return VINF_SUCCESS;
2686 }
2687
2688 Assert(pVCpu);
2689 Assert(pvUser);
2690 Assert(VMMRZCallRing3IsEnabled(pVCpu));
2691 HMSVM_ASSERT_PREEMPT_SAFE();
2692
2693 VMMRZCallRing3Disable(pVCpu);
2694 Assert(VMMR0IsLogFlushDisabled(pVCpu));
2695
2696 Log4(("hmR0SvmCallRing3Callback->hmR0SvmLongJmpToRing3\n"));
2697 int rc = hmR0SvmLongJmpToRing3(pVCpu);
2698 AssertRCReturn(rc, rc);
2699
2700 VMMRZCallRing3Enable(pVCpu);
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Take necessary actions before going back to ring-3.
2707 *
2708 * An action requires us to go back to ring-3. This function does the necessary
2709 * steps before we can safely return to ring-3. This is not the same as longjmps
2710 * to ring-3, this is voluntary.
2711 *
2712 * @returns VBox status code.
2713 * @param pVM The cross context VM structure.
2714 * @param pVCpu The cross context virtual CPU structure.
2715 * @param pCtx Pointer to the guest-CPU context.
2716 * @param rcExit The reason for exiting to ring-3. Can be
2717 * VINF_VMM_UNKNOWN_RING3_CALL.
2718 */
2719static int hmR0SvmExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, int rcExit)
2720{
2721 Assert(pVM);
2722 Assert(pVCpu);
2723 Assert(pCtx);
2724 HMSVM_ASSERT_PREEMPT_SAFE();
2725
2726 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
2727 VMMRZCallRing3Disable(pVCpu);
2728 Log4(("hmR0SvmExitToRing3: VCPU[%u]: rcExit=%d LocalFF=%#RX32 GlobalFF=%#RX32\n", pVCpu->idCpu, rcExit,
2729 pVCpu->fLocalForcedActions, pVM->fGlobalForcedActions));
2730
2731 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
2732 if (pVCpu->hm.s.Event.fPending)
2733 {
2734 hmR0SvmPendingEventToTrpmTrap(pVCpu);
2735 Assert(!pVCpu->hm.s.Event.fPending);
2736 }
2737
2738 /* Sync. the necessary state for going back to ring-3. */
2739 hmR0SvmLeaveSession(pVCpu);
2740 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
2741
2742 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2743 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
2744 | CPUM_CHANGED_LDTR
2745 | CPUM_CHANGED_GDTR
2746 | CPUM_CHANGED_IDTR
2747 | CPUM_CHANGED_TR
2748 | CPUM_CHANGED_HIDDEN_SEL_REGS);
2749 if ( pVM->hm.s.fNestedPaging
2750 && CPUMIsGuestPagingEnabledEx(pCtx))
2751 {
2752 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
2753 }
2754
2755 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
2756 if (rcExit != VINF_EM_RAW_INTERRUPT)
2757 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
2758
2759 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
2760
2761 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
2762 VMMRZCallRing3RemoveNotification(pVCpu);
2763 VMMRZCallRing3Enable(pVCpu);
2764
2765 /*
2766 * If we're emulating an instruction, we shouldn't have any TRPM traps pending
2767 * and if we're injecting an event we should have a TRPM trap pending.
2768 */
2769 AssertReturnStmt(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu),
2770 pVCpu->hm.s.u32HMError = rcExit,
2771 VERR_SVM_IPE_5);
2772 AssertReturnStmt(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu),
2773 pVCpu->hm.s.u32HMError = rcExit,
2774 VERR_SVM_IPE_4);
2775
2776 return rcExit;
2777}
2778
2779
2780#ifdef VBOX_WITH_NESTED_HWVIRT
2781/**
2782 * Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
2783 * intercepts for the nested-guest.
2784 *
2785 * @param pVM The cross context VM structure.
2786 * @param pVCpu The cross context virtual CPU structure.
2787 * @param pCtx Pointer to the nested guest-CPU context.
2788 * @param pVmcbNstGst Pointer to the nested-guest VM control block.
2789 *
2790 * @remarks No-long-jump zone!!!
2791 */
2792static void hmR0SvmUpdateTscOffsettingNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcbNstGst)
2793{
2794 Assert(CPUMIsGuestInSvmNestedHwVirtMode(pCtx));
2795
2796 bool fParavirtTsc;
2797 uint64_t uTscOffset;
2798 bool fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
2799
2800 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
2801 PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
2802
2803 /*
2804 * Only avoid intercepting if we determined the host TSC (++) is stable enough
2805 * to not intercept -and- the nested-hypervisor itself does not want to intercept it.
2806 */
2807 if ( fCanUseRealTsc
2808 && !(pVmcbNstGstCache->u64InterceptCtrl & (SVM_CTRL_INTERCEPT_RDTSC | SVM_CTRL_INTERCEPT_RDTSCP)))
2809 {
2810 pVmcbNstGstCtrl->u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_RDTSC;
2811 pVmcbNstGstCtrl->u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_RDTSCP;
2812 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
2813 }
2814 else
2815 {
2816 pVmcbNstGstCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_RDTSC;
2817 pVmcbNstGstCtrl->u64InterceptCtrl |= SVM_CTRL_INTERCEPT_RDTSCP;
2818 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
2819 }
2820
2821 /* Apply the nested-guest VMCB's TSC offset over the guest one. */
2822 uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
2823
2824 /* Update the nested-guest VMCB with the combined TSC offset (of guest and nested-guest). */
2825 pVmcbNstGstCtrl->u64TSCOffset = uTscOffset;
2826
2827 /* Finally update the VMCB clean bits since we touched the intercepts as well as the TSC offset. */
2828 pVmcbNstGstCtrl->u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2829
2830 if (fParavirtTsc)
2831 {
2832 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
2833 information before every VM-entry, hence disable it for performance sake. */
2834 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
2835 }
2836}
2837#endif
2838
2839
2840/**
2841 * Updates the use of TSC offsetting mode for the CPU and adjusts the necessary
2842 * intercepts.
2843 *
2844 * @param pVM The cross context VM structure.
2845 * @param pVCpu The cross context virtual CPU structure.
2846 * @param pVmcb Pointer to the VM control block.
2847 *
2848 * @remarks No-long-jump zone!!!
2849 */
2850static void hmR0SvmUpdateTscOffsetting(PVM pVM, PVMCPU pVCpu, PSVMVMCB pVmcb)
2851{
2852 bool fParavirtTsc;
2853 bool fCanUseRealTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVmcb->ctrl.u64TSCOffset, &fParavirtTsc);
2854 if (fCanUseRealTsc)
2855 {
2856 pVmcb->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_RDTSC;
2857 pVmcb->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_RDTSCP;
2858 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
2859 }
2860 else
2861 {
2862 pVmcb->ctrl.u64InterceptCtrl |= SVM_CTRL_INTERCEPT_RDTSC;
2863 pVmcb->ctrl.u64InterceptCtrl |= SVM_CTRL_INTERCEPT_RDTSCP;
2864 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
2865 }
2866 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
2867
2868 /** @todo later optimize this to be done elsewhere and not before every
2869 * VM-entry. */
2870 if (fParavirtTsc)
2871 {
2872 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
2873 information before every VM-entry, hence disable it for performance sake. */
2874#if 0
2875 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
2876 AssertRC(rc);
2877#endif
2878 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
2879 }
2880}
2881
2882
2883/**
2884 * Sets an event as a pending event to be injected into the guest.
2885 *
2886 * @param pVCpu The cross context virtual CPU structure.
2887 * @param pEvent Pointer to the SVM event.
2888 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
2889 * page-fault.
2890 *
2891 * @remarks Statistics counter assumes this is a guest event being reflected to
2892 * the guest i.e. 'StatInjectPendingReflect' is incremented always.
2893 */
2894DECLINLINE(void) hmR0SvmSetPendingEvent(PVMCPU pVCpu, PSVMEVENT pEvent, RTGCUINTPTR GCPtrFaultAddress)
2895{
2896 Assert(!pVCpu->hm.s.Event.fPending);
2897 Assert(pEvent->n.u1Valid);
2898
2899 pVCpu->hm.s.Event.u64IntInfo = pEvent->u;
2900 pVCpu->hm.s.Event.fPending = true;
2901 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
2902
2903 Log4(("hmR0SvmSetPendingEvent: u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u,
2904 pEvent->n.u8Vector, (uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
2905}
2906
2907
2908/**
2909 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
2910 *
2911 * @param pVCpu The cross context virtual CPU structure.
2912 */
2913DECLINLINE(void) hmR0SvmSetPendingXcptUD(PVMCPU pVCpu)
2914{
2915 SVMEVENT Event;
2916 Event.u = 0;
2917 Event.n.u1Valid = 1;
2918 Event.n.u3Type = SVM_EVENT_EXCEPTION;
2919 Event.n.u8Vector = X86_XCPT_UD;
2920 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2921}
2922
2923
2924/**
2925 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
2926 *
2927 * @param pVCpu The cross context virtual CPU structure.
2928 */
2929DECLINLINE(void) hmR0SvmSetPendingXcptDB(PVMCPU pVCpu)
2930{
2931 SVMEVENT Event;
2932 Event.u = 0;
2933 Event.n.u1Valid = 1;
2934 Event.n.u3Type = SVM_EVENT_EXCEPTION;
2935 Event.n.u8Vector = X86_XCPT_DB;
2936 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2937}
2938
2939
2940/**
2941 * Sets a page fault (\#PF) exception as pending-for-injection into the VM.
2942 *
2943 * @param pVCpu The cross context virtual CPU structure.
2944 * @param pCtx Pointer to the guest-CPU context.
2945 * @param u32ErrCode The error-code for the page-fault.
2946 * @param uFaultAddress The page fault address (CR2).
2947 *
2948 * @remarks This updates the guest CR2 with @a uFaultAddress!
2949 */
2950DECLINLINE(void) hmR0SvmSetPendingXcptPF(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t u32ErrCode, RTGCUINTPTR uFaultAddress)
2951{
2952 SVMEVENT Event;
2953 Event.u = 0;
2954 Event.n.u1Valid = 1;
2955 Event.n.u3Type = SVM_EVENT_EXCEPTION;
2956 Event.n.u8Vector = X86_XCPT_PF;
2957 Event.n.u1ErrorCodeValid = 1;
2958 Event.n.u32ErrorCode = u32ErrCode;
2959
2960 /* Update CR2 of the guest. */
2961 if (pCtx->cr2 != uFaultAddress)
2962 {
2963 pCtx->cr2 = uFaultAddress;
2964 /* The VMCB clean bit for CR2 will be updated while re-loading the guest state. */
2965 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR2);
2966 }
2967
2968 hmR0SvmSetPendingEvent(pVCpu, &Event, uFaultAddress);
2969}
2970
2971
2972/**
2973 * Sets a device-not-available (\#NM) exception as pending-for-injection into
2974 * the VM.
2975 *
2976 * @param pVCpu The cross context virtual CPU structure.
2977 */
2978DECLINLINE(void) hmR0SvmSetPendingXcptNM(PVMCPU pVCpu)
2979{
2980 SVMEVENT Event;
2981 Event.u = 0;
2982 Event.n.u1Valid = 1;
2983 Event.n.u3Type = SVM_EVENT_EXCEPTION;
2984 Event.n.u8Vector = X86_XCPT_NM;
2985 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
2986}
2987
2988
2989/**
2990 * Sets a math-fault (\#MF) exception as pending-for-injection into the VM.
2991 *
2992 * @param pVCpu The cross context virtual CPU structure.
2993 */
2994DECLINLINE(void) hmR0SvmSetPendingXcptMF(PVMCPU pVCpu)
2995{
2996 SVMEVENT Event;
2997 Event.u = 0;
2998 Event.n.u1Valid = 1;
2999 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3000 Event.n.u8Vector = X86_XCPT_MF;
3001 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3002}
3003
3004
3005/**
3006 * Sets a double fault (\#DF) exception as pending-for-injection into the VM.
3007 *
3008 * @param pVCpu The cross context virtual CPU structure.
3009 */
3010DECLINLINE(void) hmR0SvmSetPendingXcptDF(PVMCPU pVCpu)
3011{
3012 SVMEVENT Event;
3013 Event.u = 0;
3014 Event.n.u1Valid = 1;
3015 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3016 Event.n.u8Vector = X86_XCPT_DF;
3017 Event.n.u1ErrorCodeValid = 1;
3018 Event.n.u32ErrorCode = 0;
3019 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3020}
3021
3022
3023/**
3024 * Injects an event into the guest upon VMRUN by updating the relevant field
3025 * in the VMCB.
3026 *
3027 * @param pVCpu The cross context virtual CPU structure.
3028 * @param pVmcb Pointer to the guest VM control block.
3029 * @param pCtx Pointer to the guest-CPU context.
3030 * @param pEvent Pointer to the event.
3031 *
3032 * @remarks No-long-jump zone!!!
3033 * @remarks Requires CR0!
3034 */
3035DECLINLINE(void) hmR0SvmInjectEventVmcb(PVMCPU pVCpu, PSVMVMCB pVmcb, PCPUMCTX pCtx, PSVMEVENT pEvent)
3036{
3037 NOREF(pVCpu); NOREF(pCtx);
3038
3039 Assert(!pVmcb->ctrl.EventInject.n.u1Valid);
3040 pVmcb->ctrl.EventInject.u = pEvent->u;
3041 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[pEvent->n.u8Vector & MASK_INJECT_IRQ_STAT]);
3042
3043 Log4(("hmR0SvmInjectEventVmcb: u=%#RX64 u8Vector=%#x Type=%#x ErrorCodeValid=%RTbool ErrorCode=%#RX32\n", pEvent->u,
3044 pEvent->n.u8Vector, (uint8_t)pEvent->n.u3Type, !!pEvent->n.u1ErrorCodeValid, pEvent->n.u32ErrorCode));
3045}
3046
3047
3048
3049/**
3050 * Converts any TRPM trap into a pending HM event. This is typically used when
3051 * entering from ring-3 (not longjmp returns).
3052 *
3053 * @param pVCpu The cross context virtual CPU structure.
3054 */
3055static void hmR0SvmTrpmTrapToPendingEvent(PVMCPU pVCpu)
3056{
3057 Assert(TRPMHasTrap(pVCpu));
3058 Assert(!pVCpu->hm.s.Event.fPending);
3059
3060 uint8_t uVector;
3061 TRPMEVENT enmTrpmEvent;
3062 RTGCUINT uErrCode;
3063 RTGCUINTPTR GCPtrFaultAddress;
3064 uint8_t cbInstr;
3065
3066 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
3067 AssertRC(rc);
3068
3069 SVMEVENT Event;
3070 Event.u = 0;
3071 Event.n.u1Valid = 1;
3072 Event.n.u8Vector = uVector;
3073
3074 /* Refer AMD spec. 15.20 "Event Injection" for the format. */
3075 if (enmTrpmEvent == TRPM_TRAP)
3076 {
3077 Event.n.u3Type = SVM_EVENT_EXCEPTION;
3078 switch (uVector)
3079 {
3080 case X86_XCPT_NMI:
3081 {
3082 Event.n.u3Type = SVM_EVENT_NMI;
3083 break;
3084 }
3085
3086 case X86_XCPT_PF:
3087 case X86_XCPT_DF:
3088 case X86_XCPT_TS:
3089 case X86_XCPT_NP:
3090 case X86_XCPT_SS:
3091 case X86_XCPT_GP:
3092 case X86_XCPT_AC:
3093 {
3094 Event.n.u1ErrorCodeValid = 1;
3095 Event.n.u32ErrorCode = uErrCode;
3096 break;
3097 }
3098 }
3099 }
3100 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
3101 Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3102 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
3103 Event.n.u3Type = SVM_EVENT_SOFTWARE_INT;
3104 else
3105 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
3106
3107 rc = TRPMResetTrap(pVCpu);
3108 AssertRC(rc);
3109
3110 Log4(("TRPM->HM event: u=%#RX64 u8Vector=%#x uErrorCodeValid=%RTbool uErrorCode=%#RX32\n", Event.u, Event.n.u8Vector,
3111 !!Event.n.u1ErrorCodeValid, Event.n.u32ErrorCode));
3112
3113 hmR0SvmSetPendingEvent(pVCpu, &Event, GCPtrFaultAddress);
3114}
3115
3116
3117/**
3118 * Converts any pending SVM event into a TRPM trap. Typically used when leaving
3119 * AMD-V to execute any instruction.
3120 *
3121 * @param pVCpu The cross context virtual CPU structure.
3122 */
3123static void hmR0SvmPendingEventToTrpmTrap(PVMCPU pVCpu)
3124{
3125 Assert(pVCpu->hm.s.Event.fPending);
3126 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
3127
3128 SVMEVENT Event;
3129 Event.u = pVCpu->hm.s.Event.u64IntInfo;
3130
3131 uint8_t uVector = Event.n.u8Vector;
3132 uint8_t uVectorType = Event.n.u3Type;
3133 TRPMEVENT enmTrapType = HMSvmEventToTrpmEventType(&Event);
3134
3135 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, uVectorType));
3136
3137 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
3138 AssertRC(rc);
3139
3140 if (Event.n.u1ErrorCodeValid)
3141 TRPMSetErrorCode(pVCpu, Event.n.u32ErrorCode);
3142
3143 if ( uVectorType == SVM_EVENT_EXCEPTION
3144 && uVector == X86_XCPT_PF)
3145 {
3146 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
3147 Assert(pVCpu->hm.s.Event.GCPtrFaultAddress == CPUMGetGuestCR2(pVCpu));
3148 }
3149 else if (uVectorType == SVM_EVENT_SOFTWARE_INT)
3150 {
3151 AssertMsg( uVectorType == SVM_EVENT_SOFTWARE_INT
3152 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
3153 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
3154 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
3155 }
3156 pVCpu->hm.s.Event.fPending = false;
3157}
3158
3159
3160/**
3161 * Checks if the guest (or nested-guest) has an interrupt shadow active right
3162 * now.
3163 *
3164 * @returns @c true if the interrupt shadow is active, @c false otherwise.
3165 * @param pVCpu The cross context virtual CPU structure.
3166 * @param pCtx Pointer to the guest-CPU context.
3167 *
3168 * @remarks No-long-jump zone!!!
3169 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3170 */
3171DECLINLINE(bool) hmR0SvmIsIntrShadowActive(PVMCPU pVCpu, PCPUMCTX pCtx)
3172{
3173 /*
3174 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3175 * inhibit interrupts or clear any existing interrupt-inhibition.
3176 */
3177 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3178 {
3179 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3180 {
3181 /*
3182 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3183 * AMD-V, the flag's condition to be cleared is met and thus the cleared state is correct.
3184 */
3185 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3186 return false;
3187 }
3188 return true;
3189 }
3190 return false;
3191}
3192
3193
3194/**
3195 * Sets the virtual interrupt intercept control in the VMCB.
3196 *
3197 * @param pVmcb Pointer to the VM control block.
3198 */
3199DECLINLINE(void) hmR0SvmSetVirtIntrIntercept(PSVMVMCB pVmcb)
3200{
3201 /*
3202 * When AVIC isn't supported, indicate that a virtual interrupt is pending and to
3203 * cause a #VMEXIT when the guest is ready to accept interrupts. At #VMEXIT, we
3204 * then get the interrupt from the APIC (updating ISR at the right time) and
3205 * inject the interrupt.
3206 *
3207 * With AVIC is supported, we could make use of the asynchronously delivery without
3208 * #VMEXIT and we would be passing the AVIC page to SVM.
3209 */
3210 if (!(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR))
3211 {
3212 Assert(pVmcb->ctrl.IntCtrl.n.u1VIrqPending == 0);
3213 pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 1;
3214 pVmcb->ctrl.u64InterceptCtrl |= SVM_CTRL_INTERCEPT_VINTR;
3215 pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS | HMSVM_VMCB_CLEAN_TPR);
3216 Log4(("Set VINTR intercept\n"));
3217 }
3218}
3219
3220
3221/**
3222 * Clears the virtual interrupt intercept control in the VMCB as
3223 * we are figured the guest is unable process any interrupts
3224 * at this point of time.
3225 *
3226 * @param pVmcb Pointer to the VM control block.
3227 */
3228DECLINLINE(void) hmR0SvmClearVirtIntrIntercept(PSVMVMCB pVmcb)
3229{
3230 if (pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR)
3231 {
3232 Assert(pVmcb->ctrl.IntCtrl.n.u1VIrqPending == 1);
3233 pVmcb->ctrl.IntCtrl.n.u1VIrqPending = 0;
3234 pVmcb->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_VINTR;
3235 pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS | HMSVM_VMCB_CLEAN_TPR);
3236 Log4(("Cleared VINTR intercept\n"));
3237 }
3238}
3239
3240
3241/**
3242 * Sets the IRET intercept control in the VMCB which instructs AMD-V to cause a
3243 * \#VMEXIT as soon as a guest starts executing an IRET. This is used to unblock
3244 * virtual NMIs.
3245 *
3246 * @param pVmcb Pointer to the VM control block.
3247 */
3248DECLINLINE(void) hmR0SvmSetIretIntercept(PSVMVMCB pVmcb)
3249{
3250 if (!(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_IRET))
3251 {
3252 pVmcb->ctrl.u64InterceptCtrl |= SVM_CTRL_INTERCEPT_IRET;
3253 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
3254
3255 Log4(("Setting IRET intercept\n"));
3256 }
3257}
3258
3259
3260/**
3261 * Clears the IRET intercept control in the VMCB.
3262 *
3263 * @param pVmcb Pointer to the VM control block.
3264 */
3265DECLINLINE(void) hmR0SvmClearIretIntercept(PSVMVMCB pVmcb)
3266{
3267 if (pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_IRET)
3268 {
3269 pVmcb->ctrl.u64InterceptCtrl &= ~SVM_CTRL_INTERCEPT_IRET;
3270 pVmcb->ctrl.u32VmcbCleanBits &= ~(HMSVM_VMCB_CLEAN_INTERCEPTS);
3271
3272 Log4(("Clearing IRET intercept\n"));
3273 }
3274}
3275
3276#ifdef VBOX_WITH_NESTED_HWVIRT
3277
3278
3279/**
3280 * Evaluates the event to be delivered to the nested-guest and sets it as the
3281 * pending event.
3282 *
3283 * @returns VBox strict status code.
3284 * @param pVCpu The cross context virtual CPU structure.
3285 * @param pCtx Pointer to the guest-CPU context.
3286 */
3287static VBOXSTRICTRC hmR0SvmEvaluatePendingEventNested(PVMCPU pVCpu, PCPUMCTX pCtx)
3288{
3289 Log4Func(("\n"));
3290
3291 Assert(!pVCpu->hm.s.Event.fPending);
3292
3293 bool const fGif = pCtx->hwvirt.svm.fGif;
3294 if (fGif)
3295 {
3296 PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
3297
3298 bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3299
3300 /*
3301 * Check if the nested-guest can receive NMIs.
3302 * NMIs are higher priority than regular interrupts.
3303 */
3304 /** @todo SMI. SMIs take priority over NMIs. */
3305 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI))
3306 {
3307 bool const fBlockNmi = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS);
3308 if (fBlockNmi)
3309 hmR0SvmSetIretIntercept(pVmcbNstGst);
3310 else if (fIntShadow)
3311 {
3312 /** @todo Figure this out, how we shall manage virt. intercept if the
3313 * nested-guest already has one set and/or if we really need it? */
3314 //hmR0SvmSetVirtIntrIntercept(pVmcbNstGst);
3315 }
3316 else
3317 {
3318 Log4(("Pending NMI\n"));
3319
3320 SVMEVENT Event;
3321 Event.u = 0;
3322 Event.n.u1Valid = 1;
3323 Event.n.u8Vector = X86_XCPT_NMI;
3324 Event.n.u3Type = SVM_EVENT_NMI;
3325
3326 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3327 hmR0SvmSetIretIntercept(pVmcbNstGst);
3328 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3329 return VINF_SUCCESS;
3330 }
3331 }
3332
3333 /*
3334 * Check if the nested-guest can receive external interrupts (generated by
3335 * the guest's PIC/APIC).
3336 *
3337 * External intercepts, NMI, SMI etc. from the physical CPU are -always- intercepted
3338 * when executing using hardware-assisted SVM, see HMSVM_MANDATORY_GUEST_CTRL_INTERCEPTS.
3339 *
3340 * External interrupts that are generated for the outer guest may be intercepted
3341 * depending on how the nested-guest VMCB was programmed by guest software.
3342 *
3343 * Physical interrupts always take priority over virtual interrupts,
3344 * see AMD spec. 15.21.4 "Injecting Virtual (INTR) Interrupts".
3345 */
3346 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
3347 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
3348 && !fIntShadow
3349 && !pVCpu->hm.s.fSingleInstruction
3350 && CPUMCanSvmNstGstTakePhysIntr(pVCpu, pCtx))
3351 {
3352 if (pVmcbNstGstCache->u64InterceptCtrl & SVM_CTRL_INTERCEPT_INTR)
3353 {
3354 Log4(("Intercepting external interrupt -> #VMEXIT\n"));
3355 return IEMExecSvmVmexit(pVCpu, SVM_EXIT_INTR, 0, 0);
3356 }
3357
3358 uint8_t u8Interrupt;
3359 int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3360 if (RT_SUCCESS(rc))
3361 {
3362 Log4(("Injecting external interrupt u8Interrupt=%#x\n", u8Interrupt));
3363
3364 SVMEVENT Event;
3365 Event.u = 0;
3366 Event.n.u1Valid = 1;
3367 Event.n.u8Vector = u8Interrupt;
3368 Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3369
3370 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3371 }
3372 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3373 {
3374 /*
3375 * AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3376 * updated eventually when the TPR is written by the guest.
3377 */
3378 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3379 }
3380 else
3381 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3382 }
3383
3384 /*
3385 * Check if the nested-guest is intercepting virtual (using V_IRQ and related fields)
3386 * interrupt injection. The virtual interrupt injection itself, if any, will be done
3387 * by the physical CPU.
3388 */
3389 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NESTED_GUEST)
3390 && (pVmcbNstGstCache->u64InterceptCtrl & SVM_CTRL_INTERCEPT_VINTR)
3391 && CPUMCanSvmNstGstTakeVirtIntr(pCtx))
3392 {
3393 Log4(("Intercepting virtual interrupt -> #VMEXIT\n"));
3394 return IEMExecSvmVmexit(pVCpu, SVM_EXIT_VINTR, 0, 0);
3395 }
3396 }
3397
3398 return VINF_SUCCESS;
3399}
3400#endif
3401
3402
3403/**
3404 * Evaluates the event to be delivered to the guest and sets it as the pending
3405 * event.
3406 *
3407 * @param pVCpu The cross context virtual CPU structure.
3408 * @param pCtx Pointer to the guest-CPU context.
3409 *
3410 * @remarks Don't use this function when we are actively executing a
3411 * nested-guest, use hmR0SvmEvaluatePendingEventNested instead.
3412 */
3413static void hmR0SvmEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
3414{
3415 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3416 Assert(!pVCpu->hm.s.Event.fPending);
3417
3418#ifdef VBOX_WITH_NESTED_HWVIRT
3419 bool const fGif = pCtx->hwvirt.svm.fGif;
3420#else
3421 bool const fGif = true;
3422#endif
3423 Log4Func(("fGif=%RTbool\n", fGif));
3424
3425 /*
3426 * If the global interrupt flag (GIF) isn't set, even NMIs and other events are blocked.
3427 * See AMD spec. Table 15-10. "Effect of the GIF on Interrupt Handling".
3428 */
3429 if (fGif)
3430 {
3431 bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3432 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
3433 bool const fBlockNmi = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS);
3434 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
3435
3436 Log4Func(("fGif=%RTbool fBlockInt=%RTbool fIntShadow=%RTbool APIC/PIC_Pending=%RTbool\n", fGif, fBlockInt, fIntShadow,
3437 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
3438
3439 /** @todo SMI. SMIs take priority over NMIs. */
3440 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
3441 {
3442 if (fBlockNmi)
3443 hmR0SvmSetIretIntercept(pVmcb);
3444 else if (fIntShadow)
3445 hmR0SvmSetVirtIntrIntercept(pVmcb);
3446 else
3447 {
3448 Log4(("Pending NMI\n"));
3449
3450 SVMEVENT Event;
3451 Event.u = 0;
3452 Event.n.u1Valid = 1;
3453 Event.n.u8Vector = X86_XCPT_NMI;
3454 Event.n.u3Type = SVM_EVENT_NMI;
3455
3456 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3457 hmR0SvmSetIretIntercept(pVmcb);
3458 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
3459 return;
3460 }
3461 }
3462 else if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)
3463 && !pVCpu->hm.s.fSingleInstruction)
3464 {
3465 /*
3466 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
3467 * a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
3468 */
3469 if ( !fBlockInt
3470 && !fIntShadow)
3471 {
3472 uint8_t u8Interrupt;
3473 int rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
3474 if (RT_SUCCESS(rc))
3475 {
3476 Log4(("Injecting external interrupt u8Interrupt=%#x\n", u8Interrupt));
3477
3478 SVMEVENT Event;
3479 Event.u = 0;
3480 Event.n.u1Valid = 1;
3481 Event.n.u8Vector = u8Interrupt;
3482 Event.n.u3Type = SVM_EVENT_EXTERNAL_IRQ;
3483
3484 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
3485 }
3486 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
3487 {
3488 /*
3489 * AMD-V has no TPR thresholding feature. TPR and the force-flag will be
3490 * updated eventually when the TPR is written by the guest.
3491 */
3492 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
3493 }
3494 else
3495 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
3496 }
3497 else
3498 hmR0SvmSetVirtIntrIntercept(pVmcb);
3499 }
3500 }
3501}
3502
3503
3504/**
3505 * Injects any pending events into the guest or nested-guest.
3506 *
3507 * @param pVCpu The cross context virtual CPU structure.
3508 * @param pCtx Pointer to the guest-CPU context.
3509 * @param pVmcb Pointer to the VM control block.
3510 */
3511static void hmR0SvmInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMVMCB pVmcb)
3512{
3513 Assert(!TRPMHasTrap(pVCpu));
3514 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
3515
3516 bool const fIsNestedGuest = CPUMIsGuestInSvmNestedHwVirtMode(pCtx);
3517 bool const fIntShadow = hmR0SvmIsIntrShadowActive(pVCpu, pCtx);
3518 bool const fBlockInt = !fIsNestedGuest ? !(pCtx->eflags.u32 & X86_EFL_IF) : CPUMCanSvmNstGstTakePhysIntr(pVCpu, pCtx);
3519
3520 if (pVCpu->hm.s.Event.fPending)
3521 {
3522 SVMEVENT Event;
3523 Event.u = pVCpu->hm.s.Event.u64IntInfo;
3524 Assert(Event.n.u1Valid);
3525
3526 /*
3527 * Validate event injection pre-conditions.
3528 */
3529 if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3530 {
3531 Assert(!fBlockInt);
3532 Assert(!fIntShadow);
3533 }
3534 else if (Event.n.u3Type == SVM_EVENT_NMI)
3535 Assert(!fIntShadow);
3536 NOREF(fBlockInt);
3537
3538 /*
3539 * Inject it (update VMCB for injection by the hardware).
3540 */
3541 Log4(("Injecting pending HM event\n"));
3542 hmR0SvmInjectEventVmcb(pVCpu, pVmcb, pCtx, &Event);
3543 pVCpu->hm.s.Event.fPending = false;
3544
3545 if (Event.n.u3Type == SVM_EVENT_EXTERNAL_IRQ)
3546 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
3547 else
3548 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
3549 }
3550 else
3551 {
3552#ifdef VBOX_WITH_NESTED_HWVIRT
3553 /*
3554 * If IEM emulated VMRUN and injected an event, it would not clear the EVENTINJ::Valid bit
3555 * as a physical CPU clears it in the VMCB as part of the #VMEXIT (if the AMD spec. is to
3556 * believed, real behavior might differ). Regardless, IEM does it only on #VMEXIT for now
3557 * and since we are continuing nested-guest execution using hardware-assisted SVM, we need
3558 * to clear this field otherwise we will inject the event twice, see @bugref{7243#78}.
3559 */
3560 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3561 pVmcb->ctrl.EventInject.n.u1Valid = 0;
3562#endif
3563 Assert(pVmcb->ctrl.EventInject.n.u1Valid == 0);
3564 }
3565
3566 /*
3567 * Update the guest interrupt shadow in the guest or nested-guest VMCB.
3568 *
3569 * For nested-guests: We need to update it too for the scenario where IEM executes
3570 * the nested-guest but execution later continues here with an interrupt shadow active.
3571 */
3572 pVmcb->ctrl.IntShadow.n.u1IntShadow = fIntShadow;
3573}
3574
3575
3576/**
3577 * Reports world-switch error and dumps some useful debug info.
3578 *
3579 * @param pVM The cross context VM structure.
3580 * @param pVCpu The cross context virtual CPU structure.
3581 * @param rcVMRun The return code from VMRUN (or
3582 * VERR_SVM_INVALID_GUEST_STATE for invalid
3583 * guest-state).
3584 * @param pCtx Pointer to the guest-CPU context.
3585 */
3586static void hmR0SvmReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx)
3587{
3588 NOREF(pCtx);
3589 HMSVM_ASSERT_PREEMPT_SAFE();
3590 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3591 PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
3592
3593 if (rcVMRun == VERR_SVM_INVALID_GUEST_STATE)
3594 {
3595 hmR0DumpRegs(pVM, pVCpu, pCtx); NOREF(pVM);
3596#ifdef VBOX_STRICT
3597 Log4(("ctrl.u32VmcbCleanBits %#RX32\n", pVmcb->ctrl.u32VmcbCleanBits));
3598 Log4(("ctrl.u16InterceptRdCRx %#x\n", pVmcb->ctrl.u16InterceptRdCRx));
3599 Log4(("ctrl.u16InterceptWrCRx %#x\n", pVmcb->ctrl.u16InterceptWrCRx));
3600 Log4(("ctrl.u16InterceptRdDRx %#x\n", pVmcb->ctrl.u16InterceptRdDRx));
3601 Log4(("ctrl.u16InterceptWrDRx %#x\n", pVmcb->ctrl.u16InterceptWrDRx));
3602 Log4(("ctrl.u32InterceptXcpt %#x\n", pVmcb->ctrl.u32InterceptXcpt));
3603 Log4(("ctrl.u64InterceptCtrl %#RX64\n", pVmcb->ctrl.u64InterceptCtrl));
3604 Log4(("ctrl.u64IOPMPhysAddr %#RX64\n", pVmcb->ctrl.u64IOPMPhysAddr));
3605 Log4(("ctrl.u64MSRPMPhysAddr %#RX64\n", pVmcb->ctrl.u64MSRPMPhysAddr));
3606 Log4(("ctrl.u64TSCOffset %#RX64\n", pVmcb->ctrl.u64TSCOffset));
3607
3608 Log4(("ctrl.TLBCtrl.u32ASID %#x\n", pVmcb->ctrl.TLBCtrl.n.u32ASID));
3609 Log4(("ctrl.TLBCtrl.u8TLBFlush %#x\n", pVmcb->ctrl.TLBCtrl.n.u8TLBFlush));
3610 Log4(("ctrl.TLBCtrl.u24Reserved %#x\n", pVmcb->ctrl.TLBCtrl.n.u24Reserved));
3611
3612 Log4(("ctrl.IntCtrl.u8VTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u8VTPR));
3613 Log4(("ctrl.IntCtrl.u1VIrqPending %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIrqPending));
3614 Log4(("ctrl.IntCtrl.u1VGif %#x\n", pVmcb->ctrl.IntCtrl.n.u1VGif));
3615 Log4(("ctrl.IntCtrl.u6Reserved0 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved0));
3616 Log4(("ctrl.IntCtrl.u4VIntrPrio %#x\n", pVmcb->ctrl.IntCtrl.n.u4VIntrPrio));
3617 Log4(("ctrl.IntCtrl.u1IgnoreTPR %#x\n", pVmcb->ctrl.IntCtrl.n.u1IgnoreTPR));
3618 Log4(("ctrl.IntCtrl.u3Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u3Reserved));
3619 Log4(("ctrl.IntCtrl.u1VIntrMasking %#x\n", pVmcb->ctrl.IntCtrl.n.u1VIntrMasking));
3620 Log4(("ctrl.IntCtrl.u6Reserved1 %#x\n", pVmcb->ctrl.IntCtrl.n.u6Reserved1));
3621 Log4(("ctrl.IntCtrl.u8VIntrVector %#x\n", pVmcb->ctrl.IntCtrl.n.u8VIntrVector));
3622 Log4(("ctrl.IntCtrl.u24Reserved %#x\n", pVmcb->ctrl.IntCtrl.n.u24Reserved));
3623
3624 Log4(("ctrl.IntShadow.u1IntShadow %#x\n", pVmcb->ctrl.IntShadow.n.u1IntShadow));
3625 Log4(("ctrl.IntShadow.u1GuestIntMask %#x\n", pVmcb->ctrl.IntShadow.n.u1GuestIntMask));
3626 Log4(("ctrl.u64ExitCode %#RX64\n", pVmcb->ctrl.u64ExitCode));
3627 Log4(("ctrl.u64ExitInfo1 %#RX64\n", pVmcb->ctrl.u64ExitInfo1));
3628 Log4(("ctrl.u64ExitInfo2 %#RX64\n", pVmcb->ctrl.u64ExitInfo2));
3629 Log4(("ctrl.ExitIntInfo.u8Vector %#x\n", pVmcb->ctrl.ExitIntInfo.n.u8Vector));
3630 Log4(("ctrl.ExitIntInfo.u3Type %#x\n", pVmcb->ctrl.ExitIntInfo.n.u3Type));
3631 Log4(("ctrl.ExitIntInfo.u1ErrorCodeValid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid));
3632 Log4(("ctrl.ExitIntInfo.u19Reserved %#x\n", pVmcb->ctrl.ExitIntInfo.n.u19Reserved));
3633 Log4(("ctrl.ExitIntInfo.u1Valid %#x\n", pVmcb->ctrl.ExitIntInfo.n.u1Valid));
3634 Log4(("ctrl.ExitIntInfo.u32ErrorCode %#x\n", pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
3635 Log4(("ctrl.NestedPaging.u1NestedPaging %#x\n", pVmcb->ctrl.NestedPaging.n.u1NestedPaging));
3636 Log4(("ctrl.NestedPaging.u1Sev %#x\n", pVmcb->ctrl.NestedPaging.n.u1Sev));
3637 Log4(("ctrl.NestedPaging.u1SevEs %#x\n", pVmcb->ctrl.NestedPaging.n.u1SevEs));
3638 Log4(("ctrl.EventInject.u8Vector %#x\n", pVmcb->ctrl.EventInject.n.u8Vector));
3639 Log4(("ctrl.EventInject.u3Type %#x\n", pVmcb->ctrl.EventInject.n.u3Type));
3640 Log4(("ctrl.EventInject.u1ErrorCodeValid %#x\n", pVmcb->ctrl.EventInject.n.u1ErrorCodeValid));
3641 Log4(("ctrl.EventInject.u19Reserved %#x\n", pVmcb->ctrl.EventInject.n.u19Reserved));
3642 Log4(("ctrl.EventInject.u1Valid %#x\n", pVmcb->ctrl.EventInject.n.u1Valid));
3643 Log4(("ctrl.EventInject.u32ErrorCode %#x\n", pVmcb->ctrl.EventInject.n.u32ErrorCode));
3644
3645 Log4(("ctrl.u64NestedPagingCR3 %#RX64\n", pVmcb->ctrl.u64NestedPagingCR3));
3646
3647 Log4(("ctrl.LbrVirt.u1LbrVirt %#x\n", pVmcb->ctrl.LbrVirt.n.u1LbrVirt));
3648 Log4(("ctrl.LbrVirt.u1VirtVmsaveVmload %#x\n", pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload));
3649
3650 Log4(("guest.CS.u16Sel %RTsel\n", pVmcb->guest.CS.u16Sel));
3651 Log4(("guest.CS.u16Attr %#x\n", pVmcb->guest.CS.u16Attr));
3652 Log4(("guest.CS.u32Limit %#RX32\n", pVmcb->guest.CS.u32Limit));
3653 Log4(("guest.CS.u64Base %#RX64\n", pVmcb->guest.CS.u64Base));
3654 Log4(("guest.DS.u16Sel %#RTsel\n", pVmcb->guest.DS.u16Sel));
3655 Log4(("guest.DS.u16Attr %#x\n", pVmcb->guest.DS.u16Attr));
3656 Log4(("guest.DS.u32Limit %#RX32\n", pVmcb->guest.DS.u32Limit));
3657 Log4(("guest.DS.u64Base %#RX64\n", pVmcb->guest.DS.u64Base));
3658 Log4(("guest.ES.u16Sel %RTsel\n", pVmcb->guest.ES.u16Sel));
3659 Log4(("guest.ES.u16Attr %#x\n", pVmcb->guest.ES.u16Attr));
3660 Log4(("guest.ES.u32Limit %#RX32\n", pVmcb->guest.ES.u32Limit));
3661 Log4(("guest.ES.u64Base %#RX64\n", pVmcb->guest.ES.u64Base));
3662 Log4(("guest.FS.u16Sel %RTsel\n", pVmcb->guest.FS.u16Sel));
3663 Log4(("guest.FS.u16Attr %#x\n", pVmcb->guest.FS.u16Attr));
3664 Log4(("guest.FS.u32Limit %#RX32\n", pVmcb->guest.FS.u32Limit));
3665 Log4(("guest.FS.u64Base %#RX64\n", pVmcb->guest.FS.u64Base));
3666 Log4(("guest.GS.u16Sel %RTsel\n", pVmcb->guest.GS.u16Sel));
3667 Log4(("guest.GS.u16Attr %#x\n", pVmcb->guest.GS.u16Attr));
3668 Log4(("guest.GS.u32Limit %#RX32\n", pVmcb->guest.GS.u32Limit));
3669 Log4(("guest.GS.u64Base %#RX64\n", pVmcb->guest.GS.u64Base));
3670
3671 Log4(("guest.GDTR.u32Limit %#RX32\n", pVmcb->guest.GDTR.u32Limit));
3672 Log4(("guest.GDTR.u64Base %#RX64\n", pVmcb->guest.GDTR.u64Base));
3673
3674 Log4(("guest.LDTR.u16Sel %RTsel\n", pVmcb->guest.LDTR.u16Sel));
3675 Log4(("guest.LDTR.u16Attr %#x\n", pVmcb->guest.LDTR.u16Attr));
3676 Log4(("guest.LDTR.u32Limit %#RX32\n", pVmcb->guest.LDTR.u32Limit));
3677 Log4(("guest.LDTR.u64Base %#RX64\n", pVmcb->guest.LDTR.u64Base));
3678
3679 Log4(("guest.IDTR.u32Limit %#RX32\n", pVmcb->guest.IDTR.u32Limit));
3680 Log4(("guest.IDTR.u64Base %#RX64\n", pVmcb->guest.IDTR.u64Base));
3681
3682 Log4(("guest.TR.u16Sel %RTsel\n", pVmcb->guest.TR.u16Sel));
3683 Log4(("guest.TR.u16Attr %#x\n", pVmcb->guest.TR.u16Attr));
3684 Log4(("guest.TR.u32Limit %#RX32\n", pVmcb->guest.TR.u32Limit));
3685 Log4(("guest.TR.u64Base %#RX64\n", pVmcb->guest.TR.u64Base));
3686
3687 Log4(("guest.u8CPL %#x\n", pVmcb->guest.u8CPL));
3688 Log4(("guest.u64CR0 %#RX64\n", pVmcb->guest.u64CR0));
3689 Log4(("guest.u64CR2 %#RX64\n", pVmcb->guest.u64CR2));
3690 Log4(("guest.u64CR3 %#RX64\n", pVmcb->guest.u64CR3));
3691 Log4(("guest.u64CR4 %#RX64\n", pVmcb->guest.u64CR4));
3692 Log4(("guest.u64DR6 %#RX64\n", pVmcb->guest.u64DR6));
3693 Log4(("guest.u64DR7 %#RX64\n", pVmcb->guest.u64DR7));
3694
3695 Log4(("guest.u64RIP %#RX64\n", pVmcb->guest.u64RIP));
3696 Log4(("guest.u64RSP %#RX64\n", pVmcb->guest.u64RSP));
3697 Log4(("guest.u64RAX %#RX64\n", pVmcb->guest.u64RAX));
3698 Log4(("guest.u64RFlags %#RX64\n", pVmcb->guest.u64RFlags));
3699
3700 Log4(("guest.u64SysEnterCS %#RX64\n", pVmcb->guest.u64SysEnterCS));
3701 Log4(("guest.u64SysEnterEIP %#RX64\n", pVmcb->guest.u64SysEnterEIP));
3702 Log4(("guest.u64SysEnterESP %#RX64\n", pVmcb->guest.u64SysEnterESP));
3703
3704 Log4(("guest.u64EFER %#RX64\n", pVmcb->guest.u64EFER));
3705 Log4(("guest.u64STAR %#RX64\n", pVmcb->guest.u64STAR));
3706 Log4(("guest.u64LSTAR %#RX64\n", pVmcb->guest.u64LSTAR));
3707 Log4(("guest.u64CSTAR %#RX64\n", pVmcb->guest.u64CSTAR));
3708 Log4(("guest.u64SFMASK %#RX64\n", pVmcb->guest.u64SFMASK));
3709 Log4(("guest.u64KernelGSBase %#RX64\n", pVmcb->guest.u64KernelGSBase));
3710 Log4(("guest.u64GPAT %#RX64\n", pVmcb->guest.u64GPAT));
3711 Log4(("guest.u64DBGCTL %#RX64\n", pVmcb->guest.u64DBGCTL));
3712 Log4(("guest.u64BR_FROM %#RX64\n", pVmcb->guest.u64BR_FROM));
3713 Log4(("guest.u64BR_TO %#RX64\n", pVmcb->guest.u64BR_TO));
3714 Log4(("guest.u64LASTEXCPFROM %#RX64\n", pVmcb->guest.u64LASTEXCPFROM));
3715 Log4(("guest.u64LASTEXCPTO %#RX64\n", pVmcb->guest.u64LASTEXCPTO));
3716#endif /* VBOX_STRICT */
3717 }
3718 else
3719 Log4(("hmR0SvmReportWorldSwitchError: rcVMRun=%d\n", rcVMRun));
3720
3721 NOREF(pVmcb);
3722}
3723
3724
3725/**
3726 * Check per-VM and per-VCPU force flag actions that require us to go back to
3727 * ring-3 for one reason or another.
3728 *
3729 * @returns VBox status code (information status code included).
3730 * @retval VINF_SUCCESS if we don't have any actions that require going back to
3731 * ring-3.
3732 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
3733 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
3734 * interrupts)
3735 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
3736 * all EMTs to be in ring-3.
3737 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
3738 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
3739 * to the EM loop.
3740 *
3741 * @param pVM The cross context VM structure.
3742 * @param pVCpu The cross context virtual CPU structure.
3743 * @param pCtx Pointer to the guest-CPU context.
3744 */
3745static int hmR0SvmCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3746{
3747 Assert(VMMRZCallRing3IsEnabled(pVCpu));
3748
3749 /* On AMD-V we don't need to update CR3, PAE PDPES lazily. See hmR0SvmSaveGuestState(). */
3750 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
3751 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
3752
3753 /* Update pending interrupts into the APIC's IRR. */
3754 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
3755 APICUpdatePendingInterrupts(pVCpu);
3756
3757 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
3758 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
3759 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
3760 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
3761 {
3762 /* Pending PGM C3 sync. */
3763 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3764 {
3765 int rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3766 if (rc != VINF_SUCCESS)
3767 {
3768 Log4(("hmR0SvmCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
3769 return rc;
3770 }
3771 }
3772
3773 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
3774 /* -XXX- what was that about single stepping? */
3775 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
3776 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3777 {
3778 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3779 int rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
3780 Log4(("hmR0SvmCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
3781 return rc;
3782 }
3783
3784 /* Pending VM request packets, such as hardware interrupts. */
3785 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
3786 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
3787 {
3788 Log4(("hmR0SvmCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
3789 return VINF_EM_PENDING_REQUEST;
3790 }
3791
3792 /* Pending PGM pool flushes. */
3793 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3794 {
3795 Log4(("hmR0SvmCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
3796 return VINF_PGM_POOL_FLUSH_PENDING;
3797 }
3798
3799 /* Pending DMA requests. */
3800 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
3801 {
3802 Log4(("hmR0SvmCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
3803 return VINF_EM_RAW_TO_R3;
3804 }
3805 }
3806
3807 return VINF_SUCCESS;
3808}
3809
3810
3811#ifdef VBOX_WITH_NESTED_HWVIRT
3812/**
3813 * Does the preparations before executing nested-guest code in AMD-V.
3814 *
3815 * @returns VBox status code (informational status codes included).
3816 * @retval VINF_SUCCESS if we can proceed with running the guest.
3817 * @retval VINF_* scheduling changes, we have to go back to ring-3.
3818 *
3819 * @param pVM The cross context VM structure.
3820 * @param pVCpu The cross context virtual CPU structure.
3821 * @param pCtx Pointer to the guest-CPU context.
3822 * @param pSvmTransient Pointer to the SVM transient structure.
3823 *
3824 * @remarks Same caveats regarding longjumps as hmR0SvmPreRunGuest applies.
3825 * @sa hmR0SvmPreRunGuest.
3826 */
3827static int hmR0SvmPreRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
3828{
3829 HMSVM_ASSERT_PREEMPT_SAFE();
3830
3831 if (CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
3832 {
3833#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
3834 Log2(("hmR0SvmPreRunGuest: Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
3835 return VINF_EM_RESCHEDULE_REM;
3836#endif
3837 }
3838 else
3839 return VINF_SVM_VMEXIT;
3840
3841 /* Check force flag actions that might require us to go back to ring-3. */
3842 int rc = hmR0SvmCheckForceFlags(pVM, pVCpu, pCtx);
3843 if (rc != VINF_SUCCESS)
3844 return rc;
3845
3846 if (TRPMHasTrap(pVCpu))
3847 hmR0SvmTrpmTrapToPendingEvent(pVCpu);
3848 else if (!pVCpu->hm.s.Event.fPending)
3849 {
3850 VBOXSTRICTRC rcStrict = hmR0SvmEvaluatePendingEventNested(pVCpu, pCtx);
3851 if (rcStrict != VINF_SUCCESS)
3852 return VBOXSTRICTRC_VAL(rcStrict);
3853 }
3854
3855 /*
3856 * On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
3857 * Just do it in software, see @bugref{8411}.
3858 * NB: If we could continue a task switch exit we wouldn't need to do this.
3859 */
3860 if (RT_UNLIKELY( !pVM->hm.s.svm.u32Features
3861 && pVCpu->hm.s.Event.fPending
3862 && SVM_EVENT_GET_TYPE(pVCpu->hm.s.Event.u64IntInfo) == SVM_EVENT_NMI))
3863 {
3864 return VINF_EM_RAW_INJECT_TRPM_EVENT;
3865 }
3866
3867 /*
3868 * Load the nested-guest state.
3869 */
3870 rc = hmR0SvmLoadGuestStateNested(pVCpu, pCtx);
3871 AssertRCReturn(rc, rc);
3872 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull); /** @todo Get new STAM counter for this? */
3873
3874 /* Ensure we've cached (and hopefully modified) the VMCB for execution using hardware SVM. */
3875 Assert(pCtx->hwvirt.svm.fHMCachedVmcb);
3876
3877 /*
3878 * No longjmps to ring-3 from this point on!!!
3879 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
3880 * This also disables flushing of the R0-logger instance (if any).
3881 */
3882 VMMRZCallRing3Disable(pVCpu);
3883
3884 /*
3885 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
3886 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
3887 *
3888 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
3889 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
3890 *
3891 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
3892 * executing guest code.
3893 */
3894 pSvmTransient->fEFlags = ASMIntDisableFlags();
3895 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
3896 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3897 {
3898 ASMSetFlags(pSvmTransient->fEFlags);
3899 VMMRZCallRing3Enable(pVCpu);
3900 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3901 return VINF_EM_RAW_TO_R3;
3902 }
3903 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
3904 {
3905 ASMSetFlags(pSvmTransient->fEFlags);
3906 VMMRZCallRing3Enable(pVCpu);
3907 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
3908 return VINF_EM_RAW_INTERRUPT;
3909 }
3910
3911 /*
3912 * If we are injecting an NMI, we must set VMCPU_FF_BLOCK_NMIS only when we are going to execute
3913 * guest code for certain (no exits to ring-3). Otherwise, we could re-read the flag on re-entry into
3914 * AMD-V and conclude that NMI inhibition is active when we have not even delivered the NMI.
3915 *
3916 * With VT-x, this is handled by the Guest interruptibility information VMCS field which will set the
3917 * VMCS field after actually delivering the NMI which we read on VM-exit to determine the state.
3918 */
3919 if (pVCpu->hm.s.Event.fPending)
3920 {
3921 SVMEVENT Event;
3922 Event.u = pVCpu->hm.s.Event.u64IntInfo;
3923 if ( Event.n.u1Valid
3924 && Event.n.u3Type == SVM_EVENT_NMI
3925 && Event.n.u8Vector == X86_XCPT_NMI
3926 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
3927 {
3928 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
3929 }
3930 }
3931
3932 return VINF_SUCCESS;
3933}
3934#endif
3935
3936
3937/**
3938 * Does the preparations before executing guest code in AMD-V.
3939 *
3940 * This may cause longjmps to ring-3 and may even result in rescheduling to the
3941 * recompiler. We must be cautious what we do here regarding committing
3942 * guest-state information into the VMCB assuming we assuredly execute the guest
3943 * in AMD-V. If we fall back to the recompiler after updating the VMCB and
3944 * clearing the common-state (TRPM/forceflags), we must undo those changes so
3945 * that the recompiler can (and should) use them when it resumes guest
3946 * execution. Otherwise such operations must be done when we can no longer
3947 * exit to ring-3.
3948 *
3949 * @returns VBox status code (informational status codes included).
3950 * @retval VINF_SUCCESS if we can proceed with running the guest.
3951 * @retval VINF_* scheduling changes, we have to go back to ring-3.
3952 *
3953 * @param pVM The cross context VM structure.
3954 * @param pVCpu The cross context virtual CPU structure.
3955 * @param pCtx Pointer to the guest-CPU context.
3956 * @param pSvmTransient Pointer to the SVM transient structure.
3957 */
3958static int hmR0SvmPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
3959{
3960 HMSVM_ASSERT_PREEMPT_SAFE();
3961 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
3962
3963 /* Check force flag actions that might require us to go back to ring-3. */
3964 int rc = hmR0SvmCheckForceFlags(pVM, pVCpu, pCtx);
3965 if (rc != VINF_SUCCESS)
3966 return rc;
3967
3968 if (TRPMHasTrap(pVCpu))
3969 hmR0SvmTrpmTrapToPendingEvent(pVCpu);
3970 else if (!pVCpu->hm.s.Event.fPending)
3971 hmR0SvmEvaluatePendingEvent(pVCpu, pCtx);
3972
3973 /*
3974 * On the oldest AMD-V systems, we may not get enough information to reinject an NMI.
3975 * Just do it in software, see @bugref{8411}.
3976 * NB: If we could continue a task switch exit we wouldn't need to do this.
3977 */
3978 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending && (((pVCpu->hm.s.Event.u64IntInfo >> 8) & 7) == SVM_EVENT_NMI)))
3979 if (RT_UNLIKELY(!pVM->hm.s.svm.u32Features))
3980 return VINF_EM_RAW_INJECT_TRPM_EVENT;
3981
3982#ifdef HMSVM_SYNC_FULL_GUEST_STATE
3983 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
3984#endif
3985
3986 /* Load the guest bits that are not shared with the host in any way since we can longjmp or get preempted. */
3987 rc = hmR0SvmLoadGuestState(pVM, pVCpu, pCtx);
3988 AssertRCReturn(rc, rc);
3989 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
3990
3991 /*
3992 * If we're not intercepting TPR changes in the guest, save the guest TPR before the world-switch
3993 * so we can update it on the way back if the guest changed the TPR.
3994 */
3995 if (pVCpu->hm.s.svm.fSyncVTpr)
3996 {
3997 if (pVM->hm.s.fTPRPatchingActive)
3998 pSvmTransient->u8GuestTpr = pCtx->msrLSTAR;
3999 else
4000 {
4001 PCSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
4002 pSvmTransient->u8GuestTpr = pVmcb->ctrl.IntCtrl.n.u8VTPR;
4003 }
4004 }
4005
4006 /*
4007 * No longjmps to ring-3 from this point on!!!
4008 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4009 * This also disables flushing of the R0-logger instance (if any).
4010 */
4011 VMMRZCallRing3Disable(pVCpu);
4012
4013 /*
4014 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
4015 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
4016 *
4017 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
4018 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
4019 *
4020 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
4021 * executing guest code.
4022 */
4023 pSvmTransient->fEFlags = ASMIntDisableFlags();
4024 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
4025 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
4026 {
4027 ASMSetFlags(pSvmTransient->fEFlags);
4028 VMMRZCallRing3Enable(pVCpu);
4029 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
4030 return VINF_EM_RAW_TO_R3;
4031 }
4032 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
4033 {
4034 ASMSetFlags(pSvmTransient->fEFlags);
4035 VMMRZCallRing3Enable(pVCpu);
4036 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
4037 return VINF_EM_RAW_INTERRUPT;
4038 }
4039
4040 /*
4041 * If we are injecting an NMI, we must set VMCPU_FF_BLOCK_NMIS only when we are going to execute
4042 * guest code for certain (no exits to ring-3). Otherwise, we could re-read the flag on re-entry into
4043 * AMD-V and conclude that NMI inhibition is active when we have not even delivered the NMI.
4044 *
4045 * With VT-x, this is handled by the Guest interruptibility information VMCS field which will set the
4046 * VMCS field after actually delivering the NMI which we read on VM-exit to determine the state.
4047 */
4048 if (pVCpu->hm.s.Event.fPending)
4049 {
4050 SVMEVENT Event;
4051 Event.u = pVCpu->hm.s.Event.u64IntInfo;
4052 if ( Event.n.u1Valid
4053 && Event.n.u3Type == SVM_EVENT_NMI
4054 && Event.n.u8Vector == X86_XCPT_NMI
4055 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
4056 {
4057 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
4058 }
4059 }
4060
4061 return VINF_SUCCESS;
4062}
4063
4064
4065#ifdef VBOX_WITH_NESTED_HWVIRT
4066/**
4067 * Prepares to run nested-guest code in AMD-V and we've committed to doing so. This
4068 * means there is no backing out to ring-3 or anywhere else at this point.
4069 *
4070 * @param pVM The cross context VM structure.
4071 * @param pVCpu The cross context virtual CPU structure.
4072 * @param pCtx Pointer to the guest-CPU context.
4073 * @param pSvmTransient Pointer to the SVM transient structure.
4074 *
4075 * @remarks Called with preemption disabled.
4076 * @remarks No-long-jump zone!!!
4077 */
4078static void hmR0SvmPreRunGuestCommittedNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4079{
4080 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4081 Assert(VMMR0IsLogFlushDisabled(pVCpu));
4082 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4083 HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4084
4085 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4086 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4087
4088 PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
4089 hmR0SvmInjectPendingEvent(pVCpu, pCtx, pVmcbNstGst);
4090
4091 if ( pVCpu->hm.s.fPreloadGuestFpu
4092 && !CPUMIsGuestFPUStateActive(pVCpu))
4093 {
4094 CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4095 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
4096 }
4097
4098 /* Load the state shared between host and nested-guest (FPU, debug). */
4099 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
4100 hmR0SvmLoadSharedState(pVCpu, pVmcbNstGst, pCtx);
4101
4102 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT); /* Preemption might set this, nothing to do on AMD-V. */
4103 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
4104
4105 /* Setup TSC offsetting. */
4106 RTCPUID idCurrentCpu = hmR0GetCurrentCpu()->idCpu;
4107 if ( pSvmTransient->fUpdateTscOffsetting
4108 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
4109 {
4110 hmR0SvmUpdateTscOffsettingNested(pVM, pVCpu, pCtx, pVmcbNstGst);
4111 pSvmTransient->fUpdateTscOffsetting = false;
4112 }
4113
4114 /* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4115 if (idCurrentCpu != pVCpu->hm.s.idLastCpu)
4116 pVmcbNstGst->ctrl.u32VmcbCleanBits = 0;
4117
4118 /* Store status of the shared guest-host state at the time of VMRUN. */
4119#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4120 if (CPUMIsGuestInLongModeEx(pCtx))
4121 {
4122 pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
4123 pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
4124 }
4125 else
4126#endif
4127 {
4128 pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4129 pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4130 }
4131 pSvmTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
4132
4133 /* The TLB flushing would've already been setup by the nested-hypervisor. */
4134 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4135 hmR0SvmFlushTaggedTlb(pVCpu, pCtx, pVmcbNstGst);
4136 Assert(hmR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
4137
4138 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4139
4140 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
4141 to start executing. */
4142
4143 /*
4144 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
4145 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
4146 *
4147 * This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4148 */
4149 uint8_t *pbMsrBitmap = (uint8_t *)pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
4150 if ( (pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_RDTSCP)
4151 && !(pVmcbNstGst->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4152 {
4153 hmR0SvmSetMsrPermission(pVmcbNstGst, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4154 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4155 uint64_t u64GuestTscAux = CPUMR0GetGuestTscAux(pVCpu);
4156 if (u64GuestTscAux != pVCpu->hm.s.u64HostTscAux)
4157 ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTscAux);
4158 pSvmTransient->fRestoreTscAuxMsr = true;
4159 }
4160 else
4161 {
4162 hmR0SvmSetMsrPermission(pVmcbNstGst, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4163 pSvmTransient->fRestoreTscAuxMsr = false;
4164 }
4165
4166 /*
4167 * If VMCB Clean bits isn't supported by the CPU or exposed by the guest,
4168 * mark all state-bits as dirty indicating to the CPU to re-load from VMCB.
4169 */
4170 bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pCtx);
4171 if (!fSupportsVmcbCleanBits)
4172 pVmcbNstGst->ctrl.u32VmcbCleanBits = 0;
4173}
4174#endif
4175
4176
4177/**
4178 * Prepares to run guest code in AMD-V and we've committed to doing so. This
4179 * means there is no backing out to ring-3 or anywhere else at this
4180 * point.
4181 *
4182 * @param pVM The cross context VM structure.
4183 * @param pVCpu The cross context virtual CPU structure.
4184 * @param pCtx Pointer to the guest-CPU context.
4185 * @param pSvmTransient Pointer to the SVM transient structure.
4186 *
4187 * @remarks Called with preemption disabled.
4188 * @remarks No-long-jump zone!!!
4189 */
4190static void hmR0SvmPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4191{
4192 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4193 Assert(VMMR0IsLogFlushDisabled(pVCpu));
4194 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4195 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
4196
4197 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4198 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
4199
4200 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
4201 hmR0SvmInjectPendingEvent(pVCpu, pCtx, pVmcb);
4202
4203 if ( pVCpu->hm.s.fPreloadGuestFpu
4204 && !CPUMIsGuestFPUStateActive(pVCpu))
4205 {
4206 CPUMR0LoadGuestFPU(pVM, pVCpu); /* (Ignore rc, no need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
4207 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
4208 }
4209
4210 /* Load the state shared between host and guest (FPU, debug). */
4211 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
4212 hmR0SvmLoadSharedState(pVCpu, pVmcb, pCtx);
4213
4214 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT); /* Preemption might set this, nothing to do on AMD-V. */
4215 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
4216
4217 /* Setup TSC offsetting. */
4218 RTCPUID idCurrentCpu = hmR0GetCurrentCpu()->idCpu;
4219 if ( pSvmTransient->fUpdateTscOffsetting
4220 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
4221 {
4222 hmR0SvmUpdateTscOffsetting(pVM, pVCpu, pVmcb);
4223 pSvmTransient->fUpdateTscOffsetting = false;
4224 }
4225
4226 /* If we've migrating CPUs, mark the VMCB Clean bits as dirty. */
4227 if (idCurrentCpu != pVCpu->hm.s.idLastCpu)
4228 pVmcb->ctrl.u32VmcbCleanBits = 0;
4229
4230 /* Store status of the shared guest-host state at the time of VMRUN. */
4231#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4232 if (CPUMIsGuestInLongModeEx(pCtx))
4233 {
4234 pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
4235 pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
4236 }
4237 else
4238#endif
4239 {
4240 pSvmTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
4241 pSvmTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
4242 }
4243 pSvmTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
4244
4245 /* Flush the appropriate tagged-TLB entries. */
4246 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
4247 hmR0SvmFlushTaggedTlb(pVCpu, pCtx, pVmcb);
4248 Assert(hmR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
4249
4250 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
4251
4252 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
4253 to start executing. */
4254
4255 /*
4256 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
4257 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
4258 *
4259 * This should be done -after- any RDTSCPs for obtaining the host timestamp (TM, STAM etc).
4260 */
4261 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.svm.pvMsrBitmap;
4262 if ( (pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_RDTSCP)
4263 && !(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSCP))
4264 {
4265 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_PASSTHRU_READ, SVMMSREXIT_PASSTHRU_WRITE);
4266 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
4267 uint64_t u64GuestTscAux = CPUMR0GetGuestTscAux(pVCpu);
4268 if (u64GuestTscAux != pVCpu->hm.s.u64HostTscAux)
4269 ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTscAux);
4270 pSvmTransient->fRestoreTscAuxMsr = true;
4271 }
4272 else
4273 {
4274 hmR0SvmSetMsrPermission(pVmcb, pbMsrBitmap, MSR_K8_TSC_AUX, SVMMSREXIT_INTERCEPT_READ, SVMMSREXIT_INTERCEPT_WRITE);
4275 pSvmTransient->fRestoreTscAuxMsr = false;
4276 }
4277
4278 /* If VMCB Clean bits isn't supported by the CPU, simply mark all state-bits as dirty, indicating (re)load-from-VMCB. */
4279 bool const fSupportsVmcbCleanBits = hmR0SvmSupportsVmcbCleanBits(pVCpu, pCtx);
4280 if (!fSupportsVmcbCleanBits)
4281 pVmcb->ctrl.u32VmcbCleanBits = 0;
4282}
4283
4284
4285/**
4286 * Wrapper for running the guest code in AMD-V.
4287 *
4288 * @returns VBox strict status code.
4289 * @param pVM The cross context VM structure.
4290 * @param pVCpu The cross context virtual CPU structure.
4291 * @param pCtx Pointer to the guest-CPU context.
4292 *
4293 * @remarks No-long-jump zone!!!
4294 */
4295DECLINLINE(int) hmR0SvmRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4296{
4297 /*
4298 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4299 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4300 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4301 */
4302#ifdef VBOX_WITH_KERNEL_USING_XMM
4303 return hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, pVCpu->hm.s.svm.HCPhysVmcb, pCtx, pVM, pVCpu,
4304 pVCpu->hm.s.svm.pfnVMRun);
4305#else
4306 return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, pVCpu->hm.s.svm.HCPhysVmcb, pCtx, pVM, pVCpu);
4307#endif
4308}
4309
4310
4311#ifdef VBOX_WITH_NESTED_HWVIRT
4312/**
4313 * Wrapper for running the nested-guest code in AMD-V.
4314 *
4315 * @returns VBox strict status code.
4316 * @param pVM The cross context VM structure.
4317 * @param pVCpu The cross context virtual CPU structure.
4318 * @param pCtx Pointer to the guest-CPU context.
4319 *
4320 * @remarks No-long-jump zone!!!
4321 */
4322DECLINLINE(int) hmR0SvmRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4323{
4324 /*
4325 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4326 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4327 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4328 */
4329#ifdef VBOX_WITH_KERNEL_USING_XMM
4330 return hmR0SVMRunWrapXMM(pVCpu->hm.s.svm.HCPhysVmcbHost, pCtx->hwvirt.svm.HCPhysVmcb, pCtx, pVM, pVCpu,
4331 pVCpu->hm.s.svm.pfnVMRun);
4332#else
4333 return pVCpu->hm.s.svm.pfnVMRun(pVCpu->hm.s.svm.HCPhysVmcbHost, pCtx->hwvirt.svm.HCPhysVmcb, pCtx, pVM, pVCpu);
4334#endif
4335}
4336
4337
4338/**
4339 * Performs some essential restoration of state after running nested-guest code in
4340 * AMD-V.
4341 *
4342 * @param pVM The cross context VM structure.
4343 * @param pVCpu The cross context virtual CPU structure.
4344 * @param pMixedCtx Pointer to the nested-guest-CPU context. The data maybe
4345 * out-of-sync. Make sure to update the required fields
4346 * before using them.
4347 * @param pSvmTransient Pointer to the SVM transient structure.
4348 * @param rcVMRun Return code of VMRUN.
4349 *
4350 * @remarks Called with interrupts disabled.
4351 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
4352 * unconditionally when it is safe to do so.
4353 */
4354static void hmR0SvmPostRunGuestNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient, int rcVMRun)
4355{
4356 RT_NOREF(pVM);
4357 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4358
4359 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4360 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4361
4362 /* TSC read must be done early for maximum accuracy. */
4363 PSVMVMCB pVmcbNstGst = pMixedCtx->hwvirt.svm.CTX_SUFF(pVmcb);
4364 PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4365 PCSVMNESTEDVMCBCACHE pVmcbNstGstCache = &pVCpu->hm.s.svm.NstGstVmcbCache;
4366 if (!(pVmcbNstGstCtrl->u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4367 {
4368 /*
4369 * Undo what we did in hmR0SvmUpdateTscOffsettingNested() but don't restore the
4370 * nested-guest VMCB TSC offset here. It shall eventually be restored on #VMEXIT
4371 * later by HMSvmNstGstVmExitNotify().
4372 */
4373 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVmcbNstGstCtrl->u64TSCOffset - pVmcbNstGstCache->u64TSCOffset);
4374 }
4375
4376 if (pSvmTransient->fRestoreTscAuxMsr)
4377 {
4378 uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4379 CPUMR0SetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4380 if (u64GuestTscAuxMsr != pVCpu->hm.s.u64HostTscAux)
4381 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
4382 }
4383
4384 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
4385 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
4386 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4387
4388 Assert(!(ASMGetFlags() & X86_EFL_IF));
4389 ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4390 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4391
4392 /* Mark the VMCB-state cache as unmodified by VMM. */
4393 pVmcbNstGstCtrl->u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL;
4394
4395 /* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4396 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4397 {
4398 Log4(("VMRUN failure: rcVMRun=%Rrc\n", rcVMRun));
4399 return;
4400 }
4401
4402 pSvmTransient->u64ExitCode = pVmcbNstGstCtrl->u64ExitCode; /* Save the #VMEXIT reason. */
4403 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmcbNstGstCtrl->u64ExitCode);/* Update the #VMEXIT history array. */
4404 pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4405 pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4406
4407 Assert(!pVCpu->hm.s.svm.fSyncVTpr);
4408 hmR0SvmSaveGuestState(pVCpu, pMixedCtx, pVmcbNstGst); /* Save the nested-guest state from the VMCB to the
4409 guest-CPU context. */
4410}
4411#endif
4412
4413/**
4414 * Performs some essential restoration of state after running guest code in
4415 * AMD-V.
4416 *
4417 * @param pVM The cross context VM structure.
4418 * @param pVCpu The cross context virtual CPU structure.
4419 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4420 * out-of-sync. Make sure to update the required fields
4421 * before using them.
4422 * @param pSvmTransient Pointer to the SVM transient structure.
4423 * @param rcVMRun Return code of VMRUN.
4424 *
4425 * @remarks Called with interrupts disabled.
4426 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
4427 * unconditionally when it is safe to do so.
4428 */
4429static void hmR0SvmPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PSVMTRANSIENT pSvmTransient, int rcVMRun)
4430{
4431 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
4432
4433 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
4434 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
4435
4436 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
4437 pVmcb->ctrl.u32VmcbCleanBits = HMSVM_VMCB_CLEAN_ALL; /* Mark the VMCB-state cache as unmodified by VMM. */
4438
4439 /* TSC read must be done early for maximum accuracy. */
4440 if (!(pVmcb->ctrl.u64InterceptCtrl & SVM_CTRL_INTERCEPT_RDTSC))
4441 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVmcb->ctrl.u64TSCOffset);
4442
4443 if (pSvmTransient->fRestoreTscAuxMsr)
4444 {
4445 uint64_t u64GuestTscAuxMsr = ASMRdMsr(MSR_K8_TSC_AUX);
4446 CPUMR0SetGuestTscAux(pVCpu, u64GuestTscAuxMsr);
4447 if (u64GuestTscAuxMsr != pVCpu->hm.s.u64HostTscAux)
4448 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
4449 }
4450
4451 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
4452 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
4453 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
4454
4455 Assert(!(ASMGetFlags() & X86_EFL_IF));
4456 ASMSetFlags(pSvmTransient->fEFlags); /* Enable interrupts. */
4457 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
4458
4459 /* If VMRUN failed, we can bail out early. This does -not- cover SVM_EXIT_INVALID. */
4460 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
4461 {
4462 Log4(("VMRUN failure: rcVMRun=%Rrc\n", rcVMRun));
4463 return;
4464 }
4465
4466 pSvmTransient->u64ExitCode = pVmcb->ctrl.u64ExitCode; /* Save the #VMEXIT reason. */
4467 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmcb->ctrl.u64ExitCode); /* Update the #VMEXIT history array. */
4468 pSvmTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
4469 pSvmTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
4470
4471 hmR0SvmSaveGuestState(pVCpu, pMixedCtx, pVmcb); /* Save the guest state from the VMCB to the guest-CPU context. */
4472
4473 if (RT_LIKELY(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID))
4474 {
4475 if (pVCpu->hm.s.svm.fSyncVTpr)
4476 {
4477 /* TPR patching (for 32-bit guests) uses LSTAR MSR for holding the TPR value, otherwise uses the VTPR. */
4478 if ( pVM->hm.s.fTPRPatchingActive
4479 && (pMixedCtx->msrLSTAR & 0xff) != pSvmTransient->u8GuestTpr)
4480 {
4481 int rc = APICSetTpr(pVCpu, pMixedCtx->msrLSTAR & 0xff);
4482 AssertRC(rc);
4483 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
4484 }
4485 else if (pSvmTransient->u8GuestTpr != pVmcb->ctrl.IntCtrl.n.u8VTPR)
4486 {
4487 int rc = APICSetTpr(pVCpu, pVmcb->ctrl.IntCtrl.n.u8VTPR << 4);
4488 AssertRC(rc);
4489 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
4490 }
4491 }
4492 }
4493}
4494
4495
4496/**
4497 * Runs the guest code using AMD-V.
4498 *
4499 * @returns VBox status code.
4500 * @param pVM The cross context VM structure.
4501 * @param pVCpu The cross context virtual CPU structure.
4502 * @param pCtx Pointer to the guest-CPU context.
4503 * @param pcLoops Pointer to the number of executed loops.
4504 */
4505static int hmR0SvmRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4506{
4507 uint32_t const cMaxResumeLoops = pVM->hm.s.cMaxResumeLoops;
4508 Assert(pcLoops);
4509 Assert(*pcLoops <= cMaxResumeLoops);
4510
4511 SVMTRANSIENT SvmTransient;
4512 SvmTransient.fUpdateTscOffsetting = true;
4513
4514 int rc = VERR_INTERNAL_ERROR_5;
4515 for (;;)
4516 {
4517 Assert(!HMR0SuspendPending());
4518 HMSVM_ASSERT_CPU_SAFE();
4519
4520 /* Preparatory work for running guest code, this may force us to return
4521 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4522 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4523 rc = hmR0SvmPreRunGuest(pVM, pVCpu, pCtx, &SvmTransient);
4524 if (rc != VINF_SUCCESS)
4525 break;
4526
4527 /*
4528 * No longjmps to ring-3 from this point on!!!
4529 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4530 * This also disables flushing of the R0-logger instance (if any).
4531 */
4532 hmR0SvmPreRunGuestCommitted(pVM, pVCpu, pCtx, &SvmTransient);
4533 rc = hmR0SvmRunGuest(pVM, pVCpu, pCtx);
4534
4535 /* Restore any residual host-state and save any bits shared between host
4536 and guest into the guest-CPU state. Re-enables interrupts! */
4537 hmR0SvmPostRunGuest(pVM, pVCpu, pCtx, &SvmTransient, rc);
4538
4539 if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4540 || SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4541 {
4542 if (rc == VINF_SUCCESS)
4543 rc = VERR_SVM_INVALID_GUEST_STATE;
4544 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4545 hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
4546 break;
4547 }
4548
4549 /* Handle the #VMEXIT. */
4550 HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4551 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4552 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
4553 rc = hmR0SvmHandleExit(pVCpu, pCtx, &SvmTransient);
4554 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4555 if (rc != VINF_SUCCESS)
4556 break;
4557 if (++(*pcLoops) >= cMaxResumeLoops)
4558 {
4559 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4560 rc = VINF_EM_RAW_INTERRUPT;
4561 break;
4562 }
4563 }
4564
4565 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4566 return rc;
4567}
4568
4569
4570/**
4571 * Runs the guest code using AMD-V in single step mode.
4572 *
4573 * @returns VBox status code.
4574 * @param pVM The cross context VM structure.
4575 * @param pVCpu The cross context virtual CPU structure.
4576 * @param pCtx Pointer to the guest-CPU context.
4577 * @param pcLoops Pointer to the number of executed loops.
4578 */
4579static int hmR0SvmRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4580{
4581 uint32_t const cMaxResumeLoops = pVM->hm.s.cMaxResumeLoops;
4582 Assert(pcLoops);
4583 Assert(*pcLoops <= cMaxResumeLoops);
4584
4585 SVMTRANSIENT SvmTransient;
4586 SvmTransient.fUpdateTscOffsetting = true;
4587
4588 uint16_t uCsStart = pCtx->cs.Sel;
4589 uint64_t uRipStart = pCtx->rip;
4590
4591 int rc = VERR_INTERNAL_ERROR_5;
4592 for (;;)
4593 {
4594 Assert(!HMR0SuspendPending());
4595 AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
4596 ("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
4597 (unsigned)RTMpCpuId(), *pcLoops));
4598
4599 /* Preparatory work for running guest code, this may force us to return
4600 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4601 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4602 rc = hmR0SvmPreRunGuest(pVM, pVCpu, pCtx, &SvmTransient);
4603 if (rc != VINF_SUCCESS)
4604 break;
4605
4606 /*
4607 * No longjmps to ring-3 from this point on!!!
4608 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4609 * This also disables flushing of the R0-logger instance (if any).
4610 */
4611 VMMRZCallRing3Disable(pVCpu);
4612 VMMRZCallRing3RemoveNotification(pVCpu);
4613 hmR0SvmPreRunGuestCommitted(pVM, pVCpu, pCtx, &SvmTransient);
4614
4615 rc = hmR0SvmRunGuest(pVM, pVCpu, pCtx);
4616
4617 /*
4618 * Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
4619 * This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
4620 */
4621 hmR0SvmPostRunGuest(pVM, pVCpu, pCtx, &SvmTransient, rc);
4622 if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4623 || SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4624 {
4625 if (rc == VINF_SUCCESS)
4626 rc = VERR_SVM_INVALID_GUEST_STATE;
4627 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4628 hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
4629 return rc;
4630 }
4631
4632 /* Handle the #VMEXIT. */
4633 HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4634 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4635 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pVCpu->hm.s.svm.pVmcb);
4636 rc = hmR0SvmHandleExit(pVCpu, pCtx, &SvmTransient);
4637 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4638 if (rc != VINF_SUCCESS)
4639 break;
4640 if (++(*pcLoops) >= cMaxResumeLoops)
4641 {
4642 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4643 rc = VINF_EM_RAW_INTERRUPT;
4644 break;
4645 }
4646
4647 /*
4648 * Did the RIP change, if so, consider it a single step.
4649 * Otherwise, make sure one of the TFs gets set.
4650 */
4651 if ( pCtx->rip != uRipStart
4652 || pCtx->cs.Sel != uCsStart)
4653 {
4654 rc = VINF_EM_DBG_STEPPED;
4655 break;
4656 }
4657 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
4658 }
4659
4660 /*
4661 * Clear the X86_EFL_TF if necessary.
4662 */
4663 if (pVCpu->hm.s.fClearTrapFlag)
4664 {
4665 pVCpu->hm.s.fClearTrapFlag = false;
4666 pCtx->eflags.Bits.u1TF = 0;
4667 }
4668
4669 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4670 return rc;
4671}
4672
4673#ifdef VBOX_WITH_NESTED_HWVIRT
4674/**
4675 * Runs the nested-guest code using AMD-V.
4676 *
4677 * @returns VBox status code.
4678 * @param pVM The cross context VM structure.
4679 * @param pVCpu The cross context virtual CPU structure.
4680 * @param pCtx Pointer to the guest-CPU context.
4681 * @param pcLoops Pointer to the number of executed loops. If we're switching
4682 * from the guest-code execution loop to this nested-guest
4683 * execution loop pass the remainder value, else pass 0.
4684 */
4685static int hmR0SvmRunGuestCodeNested(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t *pcLoops)
4686{
4687 HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4688 Assert(pcLoops);
4689 Assert(*pcLoops <= pVM->hm.s.cMaxResumeLoops);
4690
4691 SVMTRANSIENT SvmTransient;
4692 SvmTransient.fUpdateTscOffsetting = true;
4693
4694 int rc = VERR_INTERNAL_ERROR_4;
4695 for (;;)
4696 {
4697 Assert(!HMR0SuspendPending());
4698 HMSVM_ASSERT_CPU_SAFE();
4699
4700 /* Preparatory work for running nested-guest code, this may force us to return
4701 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
4702 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
4703 rc = hmR0SvmPreRunGuestNested(pVM, pVCpu, pCtx, &SvmTransient);
4704 if (rc != VINF_SUCCESS)
4705 break;
4706
4707 /*
4708 * No longjmps to ring-3 from this point on!!!
4709 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
4710 * This also disables flushing of the R0-logger instance (if any).
4711 */
4712 hmR0SvmPreRunGuestCommittedNested(pVM, pVCpu, pCtx, &SvmTransient);
4713
4714 rc = hmR0SvmRunGuestNested(pVM, pVCpu, pCtx);
4715
4716 /* Restore any residual host-state and save any bits shared between host
4717 and guest into the guest-CPU state. Re-enables interrupts! */
4718 hmR0SvmPostRunGuestNested(pVM, pVCpu, pCtx, &SvmTransient, rc);
4719
4720 /** @todo This needs some work... we probably should cause a \#VMEXIT on
4721 * SVM_EXIT_INVALID and handle rc != VINF_SUCCESS differently. */
4722 if (RT_UNLIKELY( rc != VINF_SUCCESS /* Check for VMRUN errors. */
4723 || SvmTransient.u64ExitCode == SVM_EXIT_INVALID)) /* Check for invalid guest-state errors. */
4724 {
4725 if (rc == VINF_SUCCESS)
4726 rc = VERR_SVM_INVALID_GUEST_STATE;
4727 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4728 hmR0SvmReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
4729 break;
4730 }
4731
4732 /* Handle the #VMEXIT. */
4733 HMSVM_EXITCODE_STAM_COUNTER_INC(SvmTransient.u64ExitCode);
4734 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
4735 VBOXVMM_R0_HMSVM_VMEXIT(pVCpu, pCtx, SvmTransient.u64ExitCode, pCtx->hwvirt.svm.CTX_SUFF(pVmcb));
4736 rc = hmR0SvmHandleExitNested(pVCpu, pCtx, &SvmTransient);
4737 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4738 if (rc != VINF_SUCCESS)
4739 break;
4740 if (++(*pcLoops) >= pVM->hm.s.cMaxResumeLoops)
4741 {
4742 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
4743 rc = VINF_EM_RAW_INTERRUPT;
4744 break;
4745 }
4746
4747 /** @todo handle single-stepping */
4748 }
4749
4750 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4751 return rc;
4752}
4753#endif
4754
4755
4756/**
4757 * Runs the guest code using AMD-V.
4758 *
4759 * @returns Strict VBox status code.
4760 * @param pVM The cross context VM structure.
4761 * @param pVCpu The cross context virtual CPU structure.
4762 * @param pCtx Pointer to the guest-CPU context.
4763 */
4764VMMR0DECL(VBOXSTRICTRC) SVMR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4765{
4766 Assert(VMMRZCallRing3IsEnabled(pVCpu));
4767 HMSVM_ASSERT_PREEMPT_SAFE();
4768 VMMRZCallRing3SetNotification(pVCpu, hmR0SvmCallRing3Callback, pCtx);
4769
4770 uint32_t cLoops = 0;
4771 int rc;
4772#ifdef VBOX_WITH_NESTED_HWVIRT
4773 if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
4774#endif
4775 {
4776 if (!pVCpu->hm.s.fSingleInstruction)
4777 rc = hmR0SvmRunGuestCodeNormal(pVM, pVCpu, pCtx, &cLoops);
4778 else
4779 rc = hmR0SvmRunGuestCodeStep(pVM, pVCpu, pCtx, &cLoops);
4780 }
4781#ifdef VBOX_WITH_NESTED_HWVIRT
4782 else
4783 {
4784 rc = VINF_SVM_VMRUN;
4785 }
4786
4787 /* Re-check the nested-guest condition here as we may be transitioning from the normal
4788 execution loop into the nested-guest, hence this is not placed in the 'else' part above. */
4789 if (rc == VINF_SVM_VMRUN)
4790 {
4791 rc = hmR0SvmRunGuestCodeNested(pVM, pVCpu, pCtx, &cLoops);
4792 if (rc == VINF_SVM_VMEXIT)
4793 rc = VINF_SUCCESS;
4794 }
4795#endif
4796
4797 /* Fixup error codes. */
4798 if (rc == VERR_EM_INTERPRETER)
4799 rc = VINF_EM_RAW_EMULATE_INSTR;
4800 else if (rc == VINF_EM_RESET)
4801 rc = VINF_EM_TRIPLE_FAULT;
4802
4803 /* Prepare to return to ring-3. This will remove longjmp notifications. */
4804 rc = hmR0SvmExitToRing3(pVM, pVCpu, pCtx, rc);
4805 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
4806 return rc;
4807}
4808
4809
4810#ifdef VBOX_WITH_NESTED_HWVIRT
4811/**
4812 * Determines whether an IOIO intercept is active for the nested-guest or not.
4813 *
4814 * @param pvIoBitmap Pointer to the nested-guest IO bitmap.
4815 * @param pIoExitInfo Pointer to the SVMIOIOEXITINFO.
4816 */
4817static bool hmR0SvmIsIoInterceptActive(void *pvIoBitmap, PSVMIOIOEXITINFO pIoExitInfo)
4818{
4819 const uint16_t u16Port = pIoExitInfo->n.u16Port;
4820 const SVMIOIOTYPE enmIoType = (SVMIOIOTYPE)pIoExitInfo->n.u1Type;
4821 const uint8_t cbReg = (pIoExitInfo->u >> SVM_IOIO_OP_SIZE_SHIFT) & 7;
4822 const uint8_t cAddrSizeBits = ((pIoExitInfo->u >> SVM_IOIO_ADDR_SIZE_SHIFT) & 7) << 4;
4823 const uint8_t iEffSeg = pIoExitInfo->n.u3SEG;
4824 const bool fRep = pIoExitInfo->n.u1REP;
4825 const bool fStrIo = pIoExitInfo->n.u1STR;
4826
4827 return HMSvmIsIOInterceptActive(pvIoBitmap, u16Port, enmIoType, cbReg, cAddrSizeBits, iEffSeg, fRep, fStrIo,
4828 NULL /* pIoExitInfo */);
4829}
4830
4831
4832/**
4833 * Handles a nested-guest \#VMEXIT (for all EXITCODE values except
4834 * SVM_EXIT_INVALID).
4835 *
4836 * @returns VBox status code (informational status codes included).
4837 * @param pVCpu The cross context virtual CPU structure.
4838 * @param pCtx Pointer to the guest-CPU context.
4839 * @param pSvmTransient Pointer to the SVM transient structure.
4840 */
4841static int hmR0SvmHandleExitNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
4842{
4843 HMSVM_ASSERT_IN_NESTED_GUEST(pCtx);
4844 Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
4845 Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
4846
4847#define HM_SVM_VMEXIT_NESTED(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2) \
4848 VBOXSTRICTRC_TODO(IEMExecSvmVmexit(a_pVCpu, a_uExitCode, a_uExitInfo1, a_uExitInfo2))
4849
4850 /*
4851 * For all the #VMEXITs here we primarily figure out if the #VMEXIT is expected
4852 * by the nested-guest. If it isn't, it should be handled by the (outer) guest.
4853 */
4854 PSVMVMCB pVmcbNstGst = pCtx->hwvirt.svm.CTX_SUFF(pVmcb);
4855 PSVMVMCBCTRL pVmcbNstGstCtrl = &pVmcbNstGst->ctrl;
4856 uint64_t const uExitCode = pVmcbNstGstCtrl->u64ExitCode;
4857 uint64_t const uExitInfo1 = pVmcbNstGstCtrl->u64ExitInfo1;
4858 uint64_t const uExitInfo2 = pVmcbNstGstCtrl->u64ExitInfo2;
4859
4860 Assert(uExitCode == pVmcbNstGstCtrl->u64ExitCode);
4861 switch (uExitCode)
4862 {
4863 case SVM_EXIT_CPUID:
4864 {
4865 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CPUID))
4866 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4867 return hmR0SvmExitCpuid(pVCpu, pCtx, pSvmTransient);
4868 }
4869
4870 case SVM_EXIT_RDTSC:
4871 {
4872 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSC))
4873 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4874 return hmR0SvmExitRdtsc(pVCpu, pCtx, pSvmTransient);
4875 }
4876
4877 case SVM_EXIT_RDTSCP:
4878 {
4879 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDTSCP))
4880 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4881 return hmR0SvmExitRdtscp(pVCpu, pCtx, pSvmTransient);
4882 }
4883
4884
4885 case SVM_EXIT_MONITOR:
4886 {
4887 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MONITOR))
4888 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4889 return hmR0SvmExitMonitor(pVCpu, pCtx, pSvmTransient);
4890 }
4891
4892 case SVM_EXIT_MWAIT:
4893 {
4894 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MWAIT))
4895 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4896 return hmR0SvmExitMwait(pVCpu, pCtx, pSvmTransient);
4897 }
4898
4899 case SVM_EXIT_HLT:
4900 {
4901 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_HLT))
4902 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4903 return hmR0SvmExitHlt(pVCpu, pCtx, pSvmTransient);
4904 }
4905
4906 case SVM_EXIT_MSR:
4907 {
4908 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_MSR_PROT))
4909 {
4910 uint32_t const idMsr = pCtx->ecx;
4911 uint16_t offMsrpm;
4912 uint32_t uMsrpmBit;
4913 int rc = HMSvmGetMsrpmOffsetAndBit(idMsr, &offMsrpm, &uMsrpmBit);
4914 if (RT_SUCCESS(rc))
4915 {
4916 void const *pvMsrBitmap = pCtx->hwvirt.svm.CTX_SUFF(pvMsrBitmap);
4917 bool const fInterceptRead = ASMBitTest(pvMsrBitmap, (offMsrpm << 3) + uMsrpmBit);
4918 bool const fInterceptWrite = ASMBitTest(pvMsrBitmap, (offMsrpm << 3) + uMsrpmBit + 1);
4919
4920 if ( (fInterceptWrite && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
4921 || (fInterceptRead && pVmcbNstGstCtrl->u64ExitInfo1 == SVM_EXIT1_MSR_READ))
4922 {
4923 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4924 }
4925 }
4926 else
4927 {
4928 /*
4929 * MSRs not covered by the MSRPM automatically cause an #VMEXIT.
4930 * See AMD-V spec. "15.11 MSR Intercepts".
4931 */
4932 Assert(rc == VERR_OUT_OF_RANGE);
4933 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4934 }
4935 }
4936 return hmR0SvmExitMsr(pVCpu, pCtx, pSvmTransient);
4937 }
4938
4939 case SVM_EXIT_IOIO:
4940 {
4941 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IOIO_PROT))
4942 {
4943 void *pvIoBitmap = pCtx->hwvirt.svm.CTX_SUFF(pvIoBitmap);
4944 SVMIOIOEXITINFO IoExitInfo;
4945 IoExitInfo.u = pVmcbNstGst->ctrl.u64ExitInfo1;
4946 bool const fIntercept = hmR0SvmIsIoInterceptActive(pvIoBitmap, &IoExitInfo);
4947 if (fIntercept)
4948 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4949 }
4950 return hmR0SvmExitIOInstr(pVCpu, pCtx, pSvmTransient);
4951 }
4952
4953 case SVM_EXIT_EXCEPTION_14: /* X86_XCPT_PF */
4954 {
4955 PVM pVM = pVCpu->CTX_SUFF(pVM);
4956 if (pVM->hm.s.fNestedPaging)
4957 {
4958 uint32_t const u32ErrCode = pVmcbNstGstCtrl->u64ExitInfo1;
4959 uint64_t const uFaultAddress = pVmcbNstGstCtrl->u64ExitInfo2;
4960
4961 /* If the nested-guest is intercepting #PFs, cause a #PF #VMEXIT. */
4962 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_PF))
4963 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, u32ErrCode, uFaultAddress);
4964
4965 /* If the nested-guest is not intercepting #PFs, forward the #PF to the nested-guest. */
4966 hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
4967 return VINF_SUCCESS;
4968 }
4969 return hmR0SvmExitXcptPFNested(pVCpu, pCtx,pSvmTransient);
4970 }
4971
4972 case SVM_EXIT_EXCEPTION_7: /* X86_XCPT_NM */
4973 {
4974 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_NM))
4975 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4976 hmR0SvmSetPendingXcptNM(pVCpu);
4977 return VINF_SUCCESS;
4978 }
4979
4980 case SVM_EXIT_EXCEPTION_6: /* X86_XCPT_UD */
4981 {
4982 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_UD))
4983 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4984 hmR0SvmSetPendingXcptUD(pVCpu);
4985 return VINF_SUCCESS;
4986 }
4987
4988 case SVM_EXIT_EXCEPTION_16: /* X86_XCPT_MF */
4989 {
4990 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_MF))
4991 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
4992 hmR0SvmSetPendingXcptMF(pVCpu);
4993 return VINF_SUCCESS;
4994 }
4995
4996 case SVM_EXIT_EXCEPTION_1: /* X86_XCPT_DB */
4997 {
4998 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_DB))
4999 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5000 return hmR0SvmNestedExitXcptDB(pVCpu, pCtx, pSvmTransient);
5001 }
5002
5003 case SVM_EXIT_EXCEPTION_17: /* X86_XCPT_AC */
5004 {
5005 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_AC))
5006 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5007 return hmR0SvmExitXcptAC(pVCpu, pCtx, pSvmTransient);
5008 }
5009
5010 case SVM_EXIT_EXCEPTION_3: /* X86_XCPT_BP */
5011 {
5012 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, X86_XCPT_BP))
5013 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5014 return hmR0SvmNestedExitXcptBP(pVCpu, pCtx, pSvmTransient);
5015 }
5016
5017 case SVM_EXIT_READ_CR0:
5018 case SVM_EXIT_READ_CR3:
5019 case SVM_EXIT_READ_CR4:
5020 {
5021 uint8_t const uCr = uExitCode - SVM_EXIT_READ_CR0;
5022 if (HMIsGuestSvmReadCRxInterceptSet(pVCpu, pCtx, uCr))
5023 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5024 return hmR0SvmExitReadCRx(pVCpu, pCtx, pSvmTransient);
5025 }
5026
5027 case SVM_EXIT_WRITE_CR0:
5028 case SVM_EXIT_WRITE_CR3:
5029 case SVM_EXIT_WRITE_CR4:
5030 case SVM_EXIT_WRITE_CR8: /** @todo Shouldn't writes to CR8 go to V_TPR instead since we run with V_INTR_MASKING set?? */
5031 {
5032 uint8_t const uCr = uExitCode - SVM_EXIT_WRITE_CR0;
5033 Log4(("hmR0SvmHandleExitNested: Write CRx: u16InterceptWrCRx=%#x u64ExitCode=%#RX64 %#x\n",
5034 pVmcbNstGstCtrl->u16InterceptWrCRx, pSvmTransient->u64ExitCode, uCr));
5035
5036 if (HMIsGuestSvmWriteCRxInterceptSet(pVCpu, pCtx, uCr))
5037 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5038 return hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient);
5039 }
5040
5041 case SVM_EXIT_PAUSE:
5042 {
5043 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_PAUSE))
5044 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5045 return hmR0SvmExitPause(pVCpu, pCtx, pSvmTransient);
5046 }
5047
5048 case SVM_EXIT_VINTR:
5049 {
5050 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VINTR))
5051 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5052 return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
5053 }
5054
5055 case SVM_EXIT_INTR:
5056 case SVM_EXIT_NMI:
5057 case SVM_EXIT_SMI:
5058 {
5059 /*
5060 * We shouldn't direct physical interrupts, NMIs, SMIs to the nested-guest.
5061 *
5062 * Although we don't intercept SMIs, the nested-guest might. Therefore, we
5063 * might get an SMI #VMEXIT here so simply ignore rather than causing a
5064 * corresponding nested-guest #VMEXIT.
5065 */
5066 return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
5067 }
5068
5069 case SVM_EXIT_FERR_FREEZE:
5070 {
5071 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_FERR_FREEZE))
5072 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5073 return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
5074 }
5075
5076 case SVM_EXIT_INVLPG:
5077 {
5078 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPG))
5079 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5080 return hmR0SvmExitInvlpg(pVCpu, pCtx, pSvmTransient);
5081 }
5082
5083 case SVM_EXIT_WBINVD:
5084 {
5085 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_WBINVD))
5086 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5087 return hmR0SvmExitWbinvd(pVCpu, pCtx, pSvmTransient);
5088 }
5089
5090 case SVM_EXIT_INVD:
5091 {
5092 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVD))
5093 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5094 return hmR0SvmExitInvd(pVCpu, pCtx, pSvmTransient);
5095 }
5096
5097 case SVM_EXIT_RDPMC:
5098 {
5099 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RDPMC))
5100 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5101 return hmR0SvmExitRdpmc(pVCpu, pCtx, pSvmTransient);
5102 }
5103
5104 default:
5105 {
5106 switch (uExitCode)
5107 {
5108 case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5109 case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5110 case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5111 case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5112 {
5113 uint8_t const uDr = uExitCode - SVM_EXIT_READ_DR0;
5114 if (HMIsGuestSvmReadDRxInterceptSet(pVCpu, pCtx, uDr))
5115 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5116 return hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
5117 }
5118
5119 case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5120 case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5121 case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5122 case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5123 {
5124 uint8_t const uDr = uExitCode - SVM_EXIT_WRITE_DR0;
5125 if (HMIsGuestSvmWriteDRxInterceptSet(pVCpu, pCtx, uDr))
5126 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5127 return hmR0SvmExitWriteDRx(pVCpu, pCtx, pSvmTransient);
5128 }
5129
5130 /* The exceptions not handled here are already handled individually above (as they occur more frequently). */
5131 case SVM_EXIT_EXCEPTION_0: /*case SVM_EXIT_EXCEPTION_1:*/ case SVM_EXIT_EXCEPTION_2:
5132 /*case SVM_EXIT_EXCEPTION_3:*/ case SVM_EXIT_EXCEPTION_4: case SVM_EXIT_EXCEPTION_5:
5133 /*case SVM_EXIT_EXCEPTION_6:*/ /*case SVM_EXIT_EXCEPTION_7:*/ case SVM_EXIT_EXCEPTION_8:
5134 case SVM_EXIT_EXCEPTION_9: case SVM_EXIT_EXCEPTION_10: case SVM_EXIT_EXCEPTION_11:
5135 case SVM_EXIT_EXCEPTION_12: case SVM_EXIT_EXCEPTION_13: /*case SVM_EXIT_EXCEPTION_14:*/
5136 case SVM_EXIT_EXCEPTION_15: case SVM_EXIT_EXCEPTION_16: /*case SVM_EXIT_EXCEPTION_17:*/
5137 case SVM_EXIT_EXCEPTION_18: case SVM_EXIT_EXCEPTION_19: case SVM_EXIT_EXCEPTION_20:
5138 case SVM_EXIT_EXCEPTION_21: case SVM_EXIT_EXCEPTION_22: case SVM_EXIT_EXCEPTION_23:
5139 case SVM_EXIT_EXCEPTION_24: case SVM_EXIT_EXCEPTION_25: case SVM_EXIT_EXCEPTION_26:
5140 case SVM_EXIT_EXCEPTION_27: case SVM_EXIT_EXCEPTION_28: case SVM_EXIT_EXCEPTION_29:
5141 case SVM_EXIT_EXCEPTION_30: case SVM_EXIT_EXCEPTION_31:
5142 {
5143 uint8_t const uVector = uExitCode - SVM_EXIT_EXCEPTION_0;
5144 if (HMIsGuestSvmXcptInterceptSet(pVCpu, pCtx, uVector))
5145 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5146 return hmR0SvmExitXcptGeneric(pVCpu, pCtx, pSvmTransient);
5147 }
5148
5149 case SVM_EXIT_XSETBV:
5150 {
5151 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_XSETBV))
5152 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5153 return hmR0SvmExitXsetbv(pVCpu, pCtx, pSvmTransient);
5154 }
5155
5156 case SVM_EXIT_TASK_SWITCH:
5157 {
5158 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_TASK_SWITCH))
5159 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5160 return hmR0SvmExitTaskSwitch(pVCpu, pCtx, pSvmTransient);
5161 }
5162
5163 case SVM_EXIT_IRET:
5164 {
5165 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_IRET))
5166 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5167 return hmR0SvmExitIret(pVCpu, pCtx, pSvmTransient);
5168 }
5169
5170 case SVM_EXIT_SHUTDOWN:
5171 {
5172 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SHUTDOWN))
5173 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5174 return hmR0SvmExitShutdown(pVCpu, pCtx, pSvmTransient);
5175 }
5176
5177 case SVM_EXIT_VMMCALL:
5178 {
5179 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMMCALL))
5180 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5181 return hmR0SvmExitVmmCall(pVCpu, pCtx, pSvmTransient);
5182 }
5183
5184 case SVM_EXIT_CLGI:
5185 {
5186 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_CLGI))
5187 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5188 return hmR0SvmExitClgi(pVCpu, pCtx, pSvmTransient);
5189 }
5190
5191 case SVM_EXIT_STGI:
5192 {
5193 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_STGI))
5194 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5195 return hmR0SvmExitStgi(pVCpu, pCtx, pSvmTransient);
5196 }
5197
5198 case SVM_EXIT_VMLOAD:
5199 {
5200 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMLOAD))
5201 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5202 return hmR0SvmExitVmload(pVCpu, pCtx, pSvmTransient);
5203 }
5204
5205 case SVM_EXIT_VMSAVE:
5206 {
5207 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMSAVE))
5208 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5209 return hmR0SvmExitVmsave(pVCpu, pCtx, pSvmTransient);
5210 }
5211
5212 case SVM_EXIT_INVLPGA:
5213 {
5214 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_INVLPGA))
5215 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5216 return hmR0SvmExitInvlpga(pVCpu, pCtx, pSvmTransient);
5217 }
5218
5219 case SVM_EXIT_VMRUN:
5220 {
5221 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_VMRUN))
5222 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5223 return hmR0SvmExitVmrun(pVCpu, pCtx, pSvmTransient);
5224 }
5225
5226 case SVM_EXIT_RSM:
5227 {
5228 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_RSM))
5229 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5230 hmR0SvmSetPendingXcptUD(pVCpu);
5231 return VINF_SUCCESS;
5232 }
5233
5234 case SVM_EXIT_SKINIT:
5235 {
5236 if (HMIsGuestSvmCtrlInterceptSet(pVCpu, pCtx, SVM_CTRL_INTERCEPT_SKINIT))
5237 return HM_SVM_VMEXIT_NESTED(pVCpu, uExitCode, uExitInfo1, uExitInfo2);
5238 hmR0SvmSetPendingXcptUD(pVCpu);
5239 return VINF_SUCCESS;
5240 }
5241
5242 case SVM_EXIT_INIT: /* We shouldn't get INIT signals while executing a nested-guest. */
5243 case SVM_EXIT_NPF: /* We don't yet support nested-paging for nested-guests, so this should never happen. */
5244 {
5245 return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
5246 }
5247
5248 default:
5249 {
5250 AssertMsgFailed(("hmR0SvmHandleExitNested: Unknown exit code %#x\n", pSvmTransient->u64ExitCode));
5251 pVCpu->hm.s.u32HMError = pSvmTransient->u64ExitCode;
5252 return VERR_SVM_UNKNOWN_EXIT;
5253 }
5254 }
5255 }
5256 }
5257 /* not reached */
5258
5259#undef HM_SVM_VMEXIT_NESTED
5260}
5261#endif
5262
5263
5264/**
5265 * Handles a guest \#VMEXIT (for all EXITCODE values except SVM_EXIT_INVALID).
5266 *
5267 * @returns VBox status code (informational status codes included).
5268 * @param pVCpu The cross context virtual CPU structure.
5269 * @param pCtx Pointer to the guest-CPU context.
5270 * @param pSvmTransient Pointer to the SVM transient structure.
5271 */
5272static int hmR0SvmHandleExit(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5273{
5274 Assert(pSvmTransient->u64ExitCode != SVM_EXIT_INVALID);
5275 Assert(pSvmTransient->u64ExitCode <= SVM_EXIT_MAX);
5276
5277 /*
5278 * The ordering of the case labels is based on most-frequently-occurring #VMEXITs for most guests under
5279 * normal workloads (for some definition of "normal").
5280 */
5281 uint64_t const uExitCode = pSvmTransient->u64ExitCode;
5282 switch (uExitCode)
5283 {
5284 case SVM_EXIT_NPF:
5285 return hmR0SvmExitNestedPF(pVCpu, pCtx, pSvmTransient);
5286
5287 case SVM_EXIT_IOIO:
5288 return hmR0SvmExitIOInstr(pVCpu, pCtx, pSvmTransient);
5289
5290 case SVM_EXIT_RDTSC:
5291 return hmR0SvmExitRdtsc(pVCpu, pCtx, pSvmTransient);
5292
5293 case SVM_EXIT_RDTSCP:
5294 return hmR0SvmExitRdtscp(pVCpu, pCtx, pSvmTransient);
5295
5296 case SVM_EXIT_CPUID:
5297 return hmR0SvmExitCpuid(pVCpu, pCtx, pSvmTransient);
5298
5299 case SVM_EXIT_EXCEPTION_14: /* X86_XCPT_PF */
5300 return hmR0SvmExitXcptPF(pVCpu, pCtx, pSvmTransient);
5301
5302 case SVM_EXIT_EXCEPTION_7: /* X86_XCPT_NM */
5303 return hmR0SvmExitXcptNM(pVCpu, pCtx, pSvmTransient);
5304
5305 case SVM_EXIT_EXCEPTION_6: /* X86_XCPT_UD */
5306 return hmR0SvmExitXcptUD(pVCpu, pCtx, pSvmTransient);
5307
5308 case SVM_EXIT_EXCEPTION_16: /* X86_XCPT_MF */
5309 return hmR0SvmExitXcptMF(pVCpu, pCtx, pSvmTransient);
5310
5311 case SVM_EXIT_EXCEPTION_1: /* X86_XCPT_DB */
5312 return hmR0SvmExitXcptDB(pVCpu, pCtx, pSvmTransient);
5313
5314 case SVM_EXIT_EXCEPTION_17: /* X86_XCPT_AC */
5315 return hmR0SvmExitXcptAC(pVCpu, pCtx, pSvmTransient);
5316
5317 case SVM_EXIT_EXCEPTION_3: /* X86_XCPT_BP */
5318 return hmR0SvmExitXcptBP(pVCpu, pCtx, pSvmTransient);
5319
5320 case SVM_EXIT_MONITOR:
5321 return hmR0SvmExitMonitor(pVCpu, pCtx, pSvmTransient);
5322
5323 case SVM_EXIT_MWAIT:
5324 return hmR0SvmExitMwait(pVCpu, pCtx, pSvmTransient);
5325
5326 case SVM_EXIT_HLT:
5327 return hmR0SvmExitHlt(pVCpu, pCtx, pSvmTransient);
5328
5329 case SVM_EXIT_READ_CR0:
5330 case SVM_EXIT_READ_CR3:
5331 case SVM_EXIT_READ_CR4:
5332 return hmR0SvmExitReadCRx(pVCpu, pCtx, pSvmTransient);
5333
5334 case SVM_EXIT_WRITE_CR0:
5335 case SVM_EXIT_WRITE_CR3:
5336 case SVM_EXIT_WRITE_CR4:
5337 case SVM_EXIT_WRITE_CR8:
5338 return hmR0SvmExitWriteCRx(pVCpu, pCtx, pSvmTransient);
5339
5340 case SVM_EXIT_PAUSE:
5341 return hmR0SvmExitPause(pVCpu, pCtx, pSvmTransient);
5342
5343 case SVM_EXIT_VMMCALL:
5344 return hmR0SvmExitVmmCall(pVCpu, pCtx, pSvmTransient);
5345
5346 case SVM_EXIT_VINTR:
5347 return hmR0SvmExitVIntr(pVCpu, pCtx, pSvmTransient);
5348
5349 case SVM_EXIT_INTR:
5350 case SVM_EXIT_FERR_FREEZE:
5351 case SVM_EXIT_NMI:
5352 return hmR0SvmExitIntr(pVCpu, pCtx, pSvmTransient);
5353
5354 case SVM_EXIT_MSR:
5355 return hmR0SvmExitMsr(pVCpu, pCtx, pSvmTransient);
5356
5357 case SVM_EXIT_INVLPG:
5358 return hmR0SvmExitInvlpg(pVCpu, pCtx, pSvmTransient);
5359
5360 case SVM_EXIT_WBINVD:
5361 return hmR0SvmExitWbinvd(pVCpu, pCtx, pSvmTransient);
5362
5363 case SVM_EXIT_INVD:
5364 return hmR0SvmExitInvd(pVCpu, pCtx, pSvmTransient);
5365
5366 case SVM_EXIT_RDPMC:
5367 return hmR0SvmExitRdpmc(pVCpu, pCtx, pSvmTransient);
5368
5369 default:
5370 {
5371 switch (pSvmTransient->u64ExitCode)
5372 {
5373 case SVM_EXIT_READ_DR0: case SVM_EXIT_READ_DR1: case SVM_EXIT_READ_DR2: case SVM_EXIT_READ_DR3:
5374 case SVM_EXIT_READ_DR6: case SVM_EXIT_READ_DR7: case SVM_EXIT_READ_DR8: case SVM_EXIT_READ_DR9:
5375 case SVM_EXIT_READ_DR10: case SVM_EXIT_READ_DR11: case SVM_EXIT_READ_DR12: case SVM_EXIT_READ_DR13:
5376 case SVM_EXIT_READ_DR14: case SVM_EXIT_READ_DR15:
5377 return hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
5378
5379 case SVM_EXIT_WRITE_DR0: case SVM_EXIT_WRITE_DR1: case SVM_EXIT_WRITE_DR2: case SVM_EXIT_WRITE_DR3:
5380 case SVM_EXIT_WRITE_DR6: case SVM_EXIT_WRITE_DR7: case SVM_EXIT_WRITE_DR8: case SVM_EXIT_WRITE_DR9:
5381 case SVM_EXIT_WRITE_DR10: case SVM_EXIT_WRITE_DR11: case SVM_EXIT_WRITE_DR12: case SVM_EXIT_WRITE_DR13:
5382 case SVM_EXIT_WRITE_DR14: case SVM_EXIT_WRITE_DR15:
5383 return hmR0SvmExitWriteDRx(pVCpu, pCtx, pSvmTransient);
5384
5385 case SVM_EXIT_XSETBV:
5386 return hmR0SvmExitXsetbv(pVCpu, pCtx, pSvmTransient);
5387
5388 case SVM_EXIT_TASK_SWITCH:
5389 return hmR0SvmExitTaskSwitch(pVCpu, pCtx, pSvmTransient);
5390
5391 case SVM_EXIT_IRET:
5392 return hmR0SvmExitIret(pVCpu, pCtx, pSvmTransient);
5393
5394 case SVM_EXIT_SHUTDOWN:
5395 return hmR0SvmExitShutdown(pVCpu, pCtx, pSvmTransient);
5396
5397 case SVM_EXIT_SMI:
5398 case SVM_EXIT_INIT:
5399 {
5400 /*
5401 * We don't intercept SMIs. As for INIT signals, it really shouldn't ever happen here.
5402 * If it ever does, we want to know about it so log the exit code and bail.
5403 */
5404 return hmR0SvmExitUnexpected(pVCpu, pCtx, pSvmTransient);
5405 }
5406
5407#ifdef VBOX_WITH_NESTED_HWVIRT
5408 case SVM_EXIT_CLGI: return hmR0SvmExitClgi(pVCpu, pCtx, pSvmTransient);
5409 case SVM_EXIT_STGI: return hmR0SvmExitStgi(pVCpu, pCtx, pSvmTransient);
5410 case SVM_EXIT_VMLOAD: return hmR0SvmExitVmload(pVCpu, pCtx, pSvmTransient);
5411 case SVM_EXIT_VMSAVE: return hmR0SvmExitVmsave(pVCpu, pCtx, pSvmTransient);
5412 case SVM_EXIT_INVLPGA: return hmR0SvmExitInvlpga(pVCpu, pCtx, pSvmTransient);
5413 case SVM_EXIT_VMRUN: return hmR0SvmExitVmrun(pVCpu, pCtx, pSvmTransient);
5414#else
5415 case SVM_EXIT_CLGI:
5416 case SVM_EXIT_STGI:
5417 case SVM_EXIT_VMLOAD:
5418 case SVM_EXIT_VMSAVE:
5419 case SVM_EXIT_INVLPGA:
5420 case SVM_EXIT_VMRUN:
5421#endif
5422 case SVM_EXIT_RSM:
5423 case SVM_EXIT_SKINIT:
5424 {
5425 hmR0SvmSetPendingXcptUD(pVCpu);
5426 return VINF_SUCCESS;
5427 }
5428
5429#ifdef HMSVM_ALWAYS_TRAP_ALL_XCPTS
5430 case SVM_EXIT_EXCEPTION_0: /* X86_XCPT_DE */
5431 /* SVM_EXIT_EXCEPTION_1: */ /* X86_XCPT_DB - Handled above. */
5432 case SVM_EXIT_EXCEPTION_2: /* X86_XCPT_NMI */
5433 /* SVM_EXIT_EXCEPTION_3: */ /* X86_XCPT_BP - Handled above. */
5434 case SVM_EXIT_EXCEPTION_4: /* X86_XCPT_OF */
5435 case SVM_EXIT_EXCEPTION_5: /* X86_XCPT_BR */
5436 /* SVM_EXIT_EXCEPTION_6: */ /* X86_XCPT_UD - Handled above. */
5437 /* SVM_EXIT_EXCEPTION_7: */ /* X86_XCPT_NM - Handled above. */
5438 case SVM_EXIT_EXCEPTION_8: /* X86_XCPT_DF */
5439 case SVM_EXIT_EXCEPTION_9: /* X86_XCPT_CO_SEG_OVERRUN */
5440 case SVM_EXIT_EXCEPTION_10: /* X86_XCPT_TS */
5441 case SVM_EXIT_EXCEPTION_11: /* X86_XCPT_NP */
5442 case SVM_EXIT_EXCEPTION_12: /* X86_XCPT_SS */
5443 case SVM_EXIT_EXCEPTION_13: /* X86_XCPT_GP */
5444 /* SVM_EXIT_EXCEPTION_14: */ /* X86_XCPT_PF - Handled above. */
5445 case SVM_EXIT_EXCEPTION_15: /* Reserved. */
5446 /* SVM_EXIT_EXCEPTION_16: */ /* X86_XCPT_MF - Handled above. */
5447 /* SVM_EXIT_EXCEPTION_17: */ /* X86_XCPT_AC - Handled above. */
5448 case SVM_EXIT_EXCEPTION_18: /* X86_XCPT_MC */
5449 case SVM_EXIT_EXCEPTION_19: /* X86_XCPT_XF */
5450 case SVM_EXIT_EXCEPTION_20: case SVM_EXIT_EXCEPTION_21: case SVM_EXIT_EXCEPTION_22:
5451 case SVM_EXIT_EXCEPTION_23: case SVM_EXIT_EXCEPTION_24: case SVM_EXIT_EXCEPTION_25:
5452 case SVM_EXIT_EXCEPTION_26: case SVM_EXIT_EXCEPTION_27: case SVM_EXIT_EXCEPTION_28:
5453 case SVM_EXIT_EXCEPTION_29: case SVM_EXIT_EXCEPTION_30: case SVM_EXIT_EXCEPTION_31:
5454 return hmR0SvmExitXcptGeneric(pVCpu, pCtx, pSvmTransient);
5455#endif /* HMSVM_ALWAYS_TRAP_ALL_XCPTS */
5456
5457 default:
5458 {
5459 AssertMsgFailed(("hmR0SvmHandleExit: Unknown exit code %#RX64\n", uExitCode));
5460 pVCpu->hm.s.u32HMError = uExitCode;
5461 return VERR_SVM_UNKNOWN_EXIT;
5462 }
5463 }
5464 }
5465 }
5466 /* not reached */
5467}
5468
5469
5470#ifdef DEBUG
5471/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
5472# define HMSVM_ASSERT_PREEMPT_CPUID_VAR() \
5473 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
5474
5475# define HMSVM_ASSERT_PREEMPT_CPUID() \
5476 do \
5477 { \
5478 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
5479 AssertMsg(idAssertCpu == idAssertCpuNow, ("SVM %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
5480 } while (0)
5481
5482# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS() \
5483 do { \
5484 AssertPtr(pVCpu); \
5485 AssertPtr(pCtx); \
5486 AssertPtr(pSvmTransient); \
5487 Assert(ASMIntAreEnabled()); \
5488 HMSVM_ASSERT_PREEMPT_SAFE(); \
5489 HMSVM_ASSERT_PREEMPT_CPUID_VAR(); \
5490 Log4Func(("vcpu[%u] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-\n", (uint32_t)pVCpu->idCpu)); \
5491 HMSVM_ASSERT_PREEMPT_SAFE(); \
5492 if (VMMR0IsLogFlushDisabled(pVCpu)) \
5493 HMSVM_ASSERT_PREEMPT_CPUID(); \
5494 } while (0)
5495#else /* Release builds */
5496# define HMSVM_VALIDATE_EXIT_HANDLER_PARAMS() do { NOREF(pVCpu); NOREF(pCtx); NOREF(pSvmTransient); } while (0)
5497#endif
5498
5499
5500/**
5501 * Worker for hmR0SvmInterpretInvlpg().
5502 *
5503 * @return VBox status code.
5504 * @param pVCpu The cross context virtual CPU structure.
5505 * @param pCpu Pointer to the disassembler state.
5506 * @param pCtx The guest CPU context.
5507 */
5508static int hmR0SvmInterpretInvlPgEx(PVMCPU pVCpu, PDISCPUSTATE pCpu, PCPUMCTX pCtx)
5509{
5510 DISQPVPARAMVAL Param1;
5511 RTGCPTR GCPtrPage;
5512
5513 int rc = DISQueryParamVal(CPUMCTX2CORE(pCtx), pCpu, &pCpu->Param1, &Param1, DISQPVWHICH_SRC);
5514 if (RT_FAILURE(rc))
5515 return VERR_EM_INTERPRETER;
5516
5517 if ( Param1.type == DISQPV_TYPE_IMMEDIATE
5518 || Param1.type == DISQPV_TYPE_ADDRESS)
5519 {
5520 if (!(Param1.flags & (DISQPV_FLAG_32 | DISQPV_FLAG_64)))
5521 return VERR_EM_INTERPRETER;
5522
5523 GCPtrPage = Param1.val.val64;
5524 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), GCPtrPage);
5525 rc = VBOXSTRICTRC_VAL(rc2);
5526 }
5527 else
5528 {
5529 Log4(("hmR0SvmInterpretInvlPgEx invalid parameter type %#x\n", Param1.type));
5530 rc = VERR_EM_INTERPRETER;
5531 }
5532
5533 return rc;
5534}
5535
5536
5537/**
5538 * Interprets INVLPG.
5539 *
5540 * @returns VBox status code.
5541 * @retval VINF_* Scheduling instructions.
5542 * @retval VERR_EM_INTERPRETER Something we can't cope with.
5543 * @retval VERR_* Fatal errors.
5544 *
5545 * @param pVM The cross context VM structure.
5546 * @param pVCpu The cross context virtual CPU structure.
5547 * @param pCtx The guest CPU context.
5548 *
5549 * @remarks Updates the RIP if the instruction was executed successfully.
5550 */
5551static int hmR0SvmInterpretInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5552{
5553 /* Only allow 32 & 64 bit code. */
5554 if (CPUMGetGuestCodeBits(pVCpu) != 16)
5555 {
5556 PDISSTATE pDis = &pVCpu->hm.s.DisState;
5557 int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
5558 if ( RT_SUCCESS(rc)
5559 && pDis->pCurInstr->uOpcode == OP_INVLPG)
5560 {
5561 rc = hmR0SvmInterpretInvlPgEx(pVCpu, pDis, pCtx);
5562 if (RT_SUCCESS(rc))
5563 pCtx->rip += pDis->cbInstr;
5564 return rc;
5565 }
5566 else
5567 Log4(("hmR0SvmInterpretInvlpg: EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
5568 }
5569 return VERR_EM_INTERPRETER;
5570}
5571
5572
5573#ifdef HMSVM_USE_IEM_EVENT_REFLECTION
5574/**
5575 * Gets the IEM exception flags for the specified SVM event.
5576 *
5577 * @returns The IEM exception flags.
5578 * @param pEvent Pointer to the SVM event.
5579 *
5580 * @remarks This function currently only constructs flags required for
5581 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g. error-code
5582 * and CR2 aspects of an exception are not included).
5583 */
5584static uint32_t hmR0SvmGetIemXcptFlags(PCSVMEVENT pEvent)
5585{
5586 uint8_t const uEventType = pEvent->n.u3Type;
5587 uint32_t fIemXcptFlags;
5588 switch (uEventType)
5589 {
5590 case SVM_EVENT_EXCEPTION:
5591 /*
5592 * Only INT3 and INTO instructions can raise #BP and #OF exceptions.
5593 * See AMD spec. Table 8-1. "Interrupt Vector Source and Cause".
5594 */
5595 if (pEvent->n.u8Vector == X86_XCPT_BP)
5596 {
5597 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_BP_INSTR;
5598 break;
5599 }
5600 if (pEvent->n.u8Vector == X86_XCPT_OF)
5601 {
5602 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_OF_INSTR;
5603 break;
5604 }
5605 /** @todo How do we distinguish ICEBP \#DB from the regular one? */
5606 RT_FALL_THRU();
5607 case SVM_EVENT_NMI:
5608 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5609 break;
5610
5611 case SVM_EVENT_EXTERNAL_IRQ:
5612 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5613 break;
5614
5615 case SVM_EVENT_SOFTWARE_INT:
5616 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5617 break;
5618
5619 default:
5620 fIemXcptFlags = 0;
5621 AssertMsgFailed(("Unexpected event type! uEventType=%#x uVector=%#x", uEventType, pEvent->n.u8Vector));
5622 break;
5623 }
5624 return fIemXcptFlags;
5625}
5626
5627#else
5628/**
5629 * Determines if an exception is a contributory exception.
5630 *
5631 * Contributory exceptions are ones which can cause double-faults unless the
5632 * original exception was a benign exception. Page-fault is intentionally not
5633 * included here as it's a conditional contributory exception.
5634 *
5635 * @returns @c true if the exception is contributory, @c false otherwise.
5636 * @param uVector The exception vector.
5637 */
5638DECLINLINE(bool) hmR0SvmIsContributoryXcpt(const uint32_t uVector)
5639{
5640 switch (uVector)
5641 {
5642 case X86_XCPT_GP:
5643 case X86_XCPT_SS:
5644 case X86_XCPT_NP:
5645 case X86_XCPT_TS:
5646 case X86_XCPT_DE:
5647 return true;
5648 default:
5649 break;
5650 }
5651 return false;
5652}
5653#endif /* HMSVM_USE_IEM_EVENT_REFLECTION */
5654
5655
5656/**
5657 * Handle a condition that occurred while delivering an event through the guest
5658 * IDT.
5659 *
5660 * @returns VBox status code (informational error codes included).
5661 * @retval VINF_SUCCESS if we should continue handling the \#VMEXIT.
5662 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
5663 * continue execution of the guest which will delivery the \#DF.
5664 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5665 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5666 *
5667 * @param pVCpu The cross context virtual CPU structure.
5668 * @param pCtx Pointer to the guest-CPU context.
5669 * @param pSvmTransient Pointer to the SVM transient structure.
5670 *
5671 * @remarks No-long-jump zone!!!
5672 */
5673static int hmR0SvmCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
5674{
5675 int rc = VINF_SUCCESS;
5676 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
5677
5678 Log4(("EXITINTINFO: Pending vectoring event %#RX64 Valid=%RTbool ErrValid=%RTbool Err=%#RX32 Type=%u Vector=%u\n",
5679 pVmcb->ctrl.ExitIntInfo.u, !!pVmcb->ctrl.ExitIntInfo.n.u1Valid, !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid,
5680 pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, pVmcb->ctrl.ExitIntInfo.n.u3Type, pVmcb->ctrl.ExitIntInfo.n.u8Vector));
5681
5682 /* See AMD spec. 15.7.3 "EXITINFO Pseudo-Code". The EXITINTINFO (if valid) contains the prior exception (IDT vector)
5683 * that was trying to be delivered to the guest which caused a #VMEXIT which was intercepted (Exit vector). */
5684 if (pVmcb->ctrl.ExitIntInfo.n.u1Valid)
5685 {
5686#ifdef HMSVM_USE_IEM_EVENT_REFLECTION
5687 IEMXCPTRAISE enmRaise;
5688 IEMXCPTRAISEINFO fRaiseInfo;
5689 bool const fExitIsHwXcpt = pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0 <= SVM_EXIT_EXCEPTION_31;
5690 uint8_t const uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
5691 if (fExitIsHwXcpt)
5692 {
5693 uint8_t const uExitVector = pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0;
5694 uint32_t const fIdtVectorFlags = hmR0SvmGetIemXcptFlags(&pVmcb->ctrl.ExitIntInfo);
5695 uint32_t const fExitVectorFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5696 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5697 }
5698 else
5699 {
5700 /*
5701 * If delivery of an event caused a #VMEXIT that is not an exception (e.g. #NPF) then we
5702 * end up here.
5703 *
5704 * If the event was:
5705 * - a software interrupt, we can re-execute the instruction which will regenerate
5706 * the event.
5707 * - an NMI, we need to clear NMI blocking and re-inject the NMI.
5708 * - a hardware exception or external interrupt, we re-inject it.
5709 */
5710 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5711 if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_SOFTWARE_INT)
5712 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5713 else
5714 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5715 }
5716
5717 switch (enmRaise)
5718 {
5719 case IEMXCPTRAISE_CURRENT_XCPT:
5720 case IEMXCPTRAISE_PREV_EVENT:
5721 {
5722 /* For software interrupts, we shall re-execute the instruction. */
5723 if (!(fRaiseInfo & IEMXCPTRAISEINFO_SOFT_INT_XCPT))
5724 {
5725 RTGCUINTPTR GCPtrFaultAddress = 0;
5726
5727 /* If we are re-injecting an NMI, clear NMI blocking. */
5728 if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
5729 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5730
5731 /* Determine a vectoring #PF condition, see comment in hmR0SvmExitXcptPF(). */
5732 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5733 pSvmTransient->fVectoringPF = true;
5734 else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION
5735 && uIdtVector == X86_XCPT_PF)
5736 {
5737 /*
5738 * If the previous exception was a #PF, we need to recover the CR2 value.
5739 * This can't happen with shadow paging.
5740 */
5741 GCPtrFaultAddress = pCtx->cr2;
5742 }
5743
5744 /*
5745 * Without nested paging, when uExitVector is #PF, CR2 value will be updated from the VMCB's
5746 * exit info. fields, if it's a guest #PF, see hmR0SvmExitXcptPF().
5747 */
5748 Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
5749 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5750 hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, GCPtrFaultAddress);
5751
5752 Log4(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32 GCPtrFaultAddress=%#RX64\n",
5753 pVmcb->ctrl.ExitIntInfo.u, RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid),
5754 pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode, GCPtrFaultAddress));
5755 }
5756 break;
5757 }
5758
5759 case IEMXCPTRAISE_REEXEC_INSTR:
5760 {
5761 Assert(rc == VINF_SUCCESS);
5762 break;
5763 }
5764
5765 case IEMXCPTRAISE_DOUBLE_FAULT:
5766 {
5767 /*
5768 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5769 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5770 */
5771 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5772 {
5773 pSvmTransient->fVectoringDoublePF = true;
5774 Assert(rc == VINF_SUCCESS);
5775 }
5776 else
5777 {
5778 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5779 hmR0SvmSetPendingXcptDF(pVCpu);
5780 rc = VINF_HM_DOUBLE_FAULT;
5781 }
5782 break;
5783 }
5784
5785 case IEMXCPTRAISE_TRIPLE_FAULT:
5786 {
5787 rc = VINF_EM_RESET;
5788 break;
5789 }
5790
5791 case IEMXCPTRAISE_CPU_HANG:
5792 {
5793 rc = VERR_EM_GUEST_CPU_HANG;
5794 break;
5795 }
5796
5797 default:
5798 {
5799 AssertMsgFailed(("hmR0SvmExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
5800 rc = VERR_SVM_IPE_2;
5801 break;
5802 }
5803 }
5804#else
5805 uint8_t uIdtVector = pVmcb->ctrl.ExitIntInfo.n.u8Vector;
5806
5807 typedef enum
5808 {
5809 SVMREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5810 SVMREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5811 SVMREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5812 SVMREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5813 SVMREFLECTXCPT_NONE /* Nothing to reflect. */
5814 } SVMREFLECTXCPT;
5815
5816 SVMREFLECTXCPT enmReflect = SVMREFLECTXCPT_NONE;
5817 bool fReflectingNmi = false;
5818 if (pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXCEPTION)
5819 {
5820 if (pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0 <= SVM_EXIT_EXCEPTION_31)
5821 {
5822 uint8_t uExitVector = (uint8_t)(pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0);
5823
5824#ifdef VBOX_STRICT
5825 if ( hmR0SvmIsContributoryXcpt(uIdtVector)
5826 && uExitVector == X86_XCPT_PF)
5827 {
5828 Log4(("IDT: Contributory #PF idCpu=%u uCR2=%#RX64\n", pVCpu->idCpu, pCtx->cr2));
5829 }
5830#endif
5831
5832 if ( uIdtVector == X86_XCPT_BP
5833 || uIdtVector == X86_XCPT_OF)
5834 {
5835 /* Ignore INT3/INTO, just re-execute. See @bugref{8357}. */
5836 }
5837 else if ( uExitVector == X86_XCPT_PF
5838 && uIdtVector == X86_XCPT_PF)
5839 {
5840 pSvmTransient->fVectoringDoublePF = true;
5841 Log4(("IDT: Vectoring double #PF uCR2=%#RX64\n", pCtx->cr2));
5842 }
5843 else if ( uExitVector == X86_XCPT_AC
5844 && uIdtVector == X86_XCPT_AC)
5845 {
5846 enmReflect = SVMREFLECTXCPT_HANG;
5847 Log4(("IDT: Nested #AC - Bad guest\n"));
5848 }
5849 else if ( (pVmcb->ctrl.u32InterceptXcpt & HMSVM_CONTRIBUTORY_XCPT_MASK)
5850 && hmR0SvmIsContributoryXcpt(uExitVector)
5851 && ( hmR0SvmIsContributoryXcpt(uIdtVector)
5852 || uIdtVector == X86_XCPT_PF))
5853 {
5854 enmReflect = SVMREFLECTXCPT_DF;
5855 Log4(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5856 uIdtVector, uExitVector));
5857 }
5858 else if (uIdtVector == X86_XCPT_DF)
5859 {
5860 enmReflect = SVMREFLECTXCPT_TF;
5861 Log4(("IDT: Pending vectoring triple-fault %#RX64 uIdtVector=%#x uExitVector=%#x\n",
5862 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5863 }
5864 else
5865 enmReflect = SVMREFLECTXCPT_XCPT;
5866 }
5867 else
5868 {
5869 /*
5870 * If event delivery caused an #VMEXIT that is not an exception (e.g. #NPF) then reflect the original
5871 * exception to the guest after handling the #VMEXIT.
5872 */
5873 enmReflect = SVMREFLECTXCPT_XCPT;
5874 }
5875 }
5876 else if ( pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_EXTERNAL_IRQ
5877 || pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI)
5878 {
5879 enmReflect = SVMREFLECTXCPT_XCPT;
5880 fReflectingNmi = RT_BOOL(pVmcb->ctrl.ExitIntInfo.n.u3Type == SVM_EVENT_NMI);
5881
5882 if (pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0 <= SVM_EXIT_EXCEPTION_31)
5883 {
5884 uint8_t uExitVector = (uint8_t)(pSvmTransient->u64ExitCode - SVM_EXIT_EXCEPTION_0);
5885 if (uExitVector == X86_XCPT_PF)
5886 {
5887 pSvmTransient->fVectoringPF = true;
5888 Log4(("IDT: Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pCtx->cr2));
5889 }
5890 }
5891 }
5892 /* else: Ignore software interrupts (INT n) as they reoccur when restarting the instruction. */
5893
5894 switch (enmReflect)
5895 {
5896 case SVMREFLECTXCPT_XCPT:
5897 {
5898 /* If we are re-injecting the NMI, clear NMI blocking. */
5899 if (fReflectingNmi)
5900 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5901
5902 Assert(pVmcb->ctrl.ExitIntInfo.n.u3Type != SVM_EVENT_SOFTWARE_INT);
5903 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5904 hmR0SvmSetPendingEvent(pVCpu, &pVmcb->ctrl.ExitIntInfo, 0 /* GCPtrFaultAddress */);
5905
5906 /* If uExitVector is #PF, CR2 value will be updated from the VMCB if it's a guest #PF. See hmR0SvmExitXcptPF(). */
5907 Log4(("IDT: Pending vectoring event %#RX64 ErrValid=%RTbool Err=%#RX32\n", pVmcb->ctrl.ExitIntInfo.u,
5908 !!pVmcb->ctrl.ExitIntInfo.n.u1ErrorCodeValid, pVmcb->ctrl.ExitIntInfo.n.u32ErrorCode));
5909 break;
5910 }
5911
5912 case SVMREFLECTXCPT_DF:
5913 {
5914 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5915 hmR0SvmSetPendingXcptDF(pVCpu);
5916 rc = VINF_HM_DOUBLE_FAULT;
5917 break;
5918 }
5919
5920 case SVMREFLECTXCPT_TF:
5921 {
5922 rc = VINF_EM_RESET;
5923 break;
5924 }
5925
5926 case SVMREFLECTXCPT_HANG:
5927 {
5928 rc = VERR_EM_GUEST_CPU_HANG;
5929 break;
5930 }
5931
5932 default:
5933 Assert(rc == VINF_SUCCESS);
5934 break;
5935 }
5936#endif /* HMSVM_USE_IEM_EVENT_REFLECTION */
5937 }
5938 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET || rc == VERR_EM_GUEST_CPU_HANG);
5939 NOREF(pCtx);
5940 return rc;
5941}
5942
5943
5944/**
5945 * Advances the guest RIP making use of the CPU's NRIP_SAVE feature if
5946 * supported, otherwise advances the RIP by the number of bytes specified in
5947 * @a cb.
5948 *
5949 * @param pVCpu The cross context virtual CPU structure.
5950 * @param pCtx Pointer to the guest-CPU context.
5951 * @param cb RIP increment value in bytes.
5952 *
5953 * @remarks Use this function only from \#VMEXIT's where the NRIP value is valid
5954 * when NRIP_SAVE is supported by the CPU, otherwise use
5955 * hmR0SvmAdvanceRipDumb!
5956 */
5957DECLINLINE(void) hmR0SvmAdvanceRipHwAssist(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t cb)
5958{
5959 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
5960 if (fSupportsNextRipSave)
5961 {
5962 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
5963 Assert(pVmcb->ctrl.u64NextRIP);
5964 AssertRelease(pVmcb->ctrl.u64NextRIP - pCtx->rip == cb); /* temporary, remove later */
5965 pCtx->rip = pVmcb->ctrl.u64NextRIP;
5966 }
5967 else
5968 pCtx->rip += cb;
5969
5970 HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx);
5971}
5972
5973
5974#ifdef VBOX_WITH_NESTED_HWVIRT
5975/**
5976 * Gets the length of the current instruction if the CPU supports the NRIP_SAVE
5977 * feature. Otherwise, returns the value in @a cbLikely.
5978 *
5979 * @param pVCpu The cross context virtual CPU structure.
5980 * @param pCtx Pointer to the guest-CPU context.
5981 * @param cbLikely The likely instruction length.
5982 */
5983DECLINLINE(uint8_t) hmR0SvmGetInstrLengthHwAssist(PVMCPU pVCpu, PCPUMCTX pCtx, uint8_t cbLikely)
5984{
5985 Assert(cbLikely <= 15); /* See Intel spec. 2.3.11 "AVX Instruction Length" */
5986 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
5987 if (fSupportsNextRipSave)
5988 {
5989 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
5990 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
5991 Assert(cbInstr == cbLikely);
5992 return cbInstr;
5993 }
5994 return cbLikely;
5995}
5996#endif
5997
5998
5999/**
6000 * Advances the guest RIP by the number of bytes specified in @a cb. This does
6001 * not make use of any hardware features to determine the instruction length.
6002 *
6003 * @param pVCpu The cross context virtual CPU structure.
6004 * @param pCtx Pointer to the guest-CPU context.
6005 * @param cb RIP increment value in bytes.
6006 */
6007DECLINLINE(void) hmR0SvmAdvanceRipDumb(PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t cb)
6008{
6009 pCtx->rip += cb;
6010 HMSVM_UPDATE_INTR_SHADOW(pVCpu, pCtx);
6011}
6012#undef HMSVM_UPDATE_INTR_SHADOW
6013
6014
6015/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6016/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- #VMEXIT handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6017/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6018
6019/** @name \#VMEXIT handlers.
6020 * @{
6021 */
6022
6023/**
6024 * \#VMEXIT handler for external interrupts, NMIs, FPU assertion freeze and INIT
6025 * signals (SVM_EXIT_INTR, SVM_EXIT_NMI, SVM_EXIT_FERR_FREEZE, SVM_EXIT_INIT).
6026 */
6027HMSVM_EXIT_DECL hmR0SvmExitIntr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6028{
6029 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6030
6031 if (pSvmTransient->u64ExitCode == SVM_EXIT_NMI)
6032 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
6033 else if (pSvmTransient->u64ExitCode == SVM_EXIT_INTR)
6034 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6035
6036 /*
6037 * AMD-V has no preemption timer and the generic periodic preemption timer has no way to signal -before- the timer
6038 * fires if the current interrupt is our own timer or a some other host interrupt. We also cannot examine what
6039 * interrupt it is until the host actually take the interrupt.
6040 *
6041 * Going back to executing guest code here unconditionally causes random scheduling problems (observed on an
6042 * AMD Phenom 9850 Quad-Core on Windows 64-bit host).
6043 */
6044 return VINF_EM_RAW_INTERRUPT;
6045}
6046
6047
6048/**
6049 * \#VMEXIT handler for WBINVD (SVM_EXIT_WBINVD). Conditional \#VMEXIT.
6050 */
6051HMSVM_EXIT_DECL hmR0SvmExitWbinvd(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6052{
6053 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6054
6055 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6056 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6057 int rc = VINF_SUCCESS;
6058 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6059 return rc;
6060}
6061
6062
6063/**
6064 * \#VMEXIT handler for INVD (SVM_EXIT_INVD). Unconditional \#VMEXIT.
6065 */
6066HMSVM_EXIT_DECL hmR0SvmExitInvd(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6067{
6068 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6069
6070 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6071 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6072 int rc = VINF_SUCCESS;
6073 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6074 return rc;
6075}
6076
6077
6078/**
6079 * \#VMEXIT handler for INVD (SVM_EXIT_CPUID). Conditional \#VMEXIT.
6080 */
6081HMSVM_EXIT_DECL hmR0SvmExitCpuid(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6082{
6083 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6084 PVM pVM = pVCpu->CTX_SUFF(pVM);
6085 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6086 if (RT_LIKELY(rc == VINF_SUCCESS))
6087 {
6088 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6089 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6090 }
6091 else
6092 {
6093 AssertMsgFailed(("hmR0SvmExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6094 rc = VERR_EM_INTERPRETER;
6095 }
6096 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6097 return rc;
6098}
6099
6100
6101/**
6102 * \#VMEXIT handler for RDTSC (SVM_EXIT_RDTSC). Conditional \#VMEXIT.
6103 */
6104HMSVM_EXIT_DECL hmR0SvmExitRdtsc(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6105{
6106 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6107 PVM pVM = pVCpu->CTX_SUFF(pVM);
6108 int rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6109 if (RT_LIKELY(rc == VINF_SUCCESS))
6110 {
6111 pSvmTransient->fUpdateTscOffsetting = true;
6112 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6113 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6114 }
6115 else
6116 {
6117 AssertMsgFailed(("hmR0SvmExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
6118 rc = VERR_EM_INTERPRETER;
6119 }
6120 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6121 return rc;
6122}
6123
6124
6125/**
6126 * \#VMEXIT handler for RDTSCP (SVM_EXIT_RDTSCP). Conditional \#VMEXIT.
6127 */
6128HMSVM_EXIT_DECL hmR0SvmExitRdtscp(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6129{
6130 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6131 int rc = EMInterpretRdtscp(pVCpu->CTX_SUFF(pVM), pVCpu, pCtx);
6132 if (RT_LIKELY(rc == VINF_SUCCESS))
6133 {
6134 pSvmTransient->fUpdateTscOffsetting = true;
6135 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6136 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6137 }
6138 else
6139 {
6140 AssertMsgFailed(("hmR0SvmExitRdtsc: EMInterpretRdtscp failed with %Rrc\n", rc));
6141 rc = VERR_EM_INTERPRETER;
6142 }
6143 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
6144 return rc;
6145}
6146
6147
6148/**
6149 * \#VMEXIT handler for RDPMC (SVM_EXIT_RDPMC). Conditional \#VMEXIT.
6150 */
6151HMSVM_EXIT_DECL hmR0SvmExitRdpmc(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6152{
6153 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6154 int rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6155 if (RT_LIKELY(rc == VINF_SUCCESS))
6156 {
6157 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6158 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6159 }
6160 else
6161 {
6162 AssertMsgFailed(("hmR0SvmExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
6163 rc = VERR_EM_INTERPRETER;
6164 }
6165 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
6166 return rc;
6167}
6168
6169
6170/**
6171 * \#VMEXIT handler for INVLPG (SVM_EXIT_INVLPG). Conditional \#VMEXIT.
6172 */
6173HMSVM_EXIT_DECL hmR0SvmExitInvlpg(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6174{
6175 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6176 PVM pVM = pVCpu->CTX_SUFF(pVM);
6177 Assert(!pVM->hm.s.fNestedPaging);
6178 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
6179
6180 bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6181 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6182 if ( fSupportsDecodeAssists
6183 && fSupportsNextRipSave)
6184 {
6185 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6186 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6187 RTGCPTR const GCPtrPage = pVmcb->ctrl.u64ExitInfo1;
6188 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, cbInstr, GCPtrPage);
6189 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6190 return VBOXSTRICTRC_VAL(rcStrict);
6191 }
6192
6193 int rc = hmR0SvmInterpretInvlpg(pVM, pVCpu, pCtx); /* Updates RIP if successful. */
6194 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
6195 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6196 return rc;
6197}
6198
6199
6200/**
6201 * \#VMEXIT handler for HLT (SVM_EXIT_HLT). Conditional \#VMEXIT.
6202 */
6203HMSVM_EXIT_DECL hmR0SvmExitHlt(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6204{
6205 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6206
6207 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 1);
6208 int rc = EMShouldContinueAfterHalt(pVCpu, pCtx) ? VINF_SUCCESS : VINF_EM_HALT;
6209 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6210 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
6211 if (rc != VINF_SUCCESS)
6212 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
6213 return rc;
6214}
6215
6216
6217/**
6218 * \#VMEXIT handler for MONITOR (SVM_EXIT_MONITOR). Conditional \#VMEXIT.
6219 */
6220HMSVM_EXIT_DECL hmR0SvmExitMonitor(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6221{
6222 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6223 int rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6224 if (RT_LIKELY(rc == VINF_SUCCESS))
6225 {
6226 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6227 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6228 }
6229 else
6230 {
6231 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0SvmExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
6232 rc = VERR_EM_INTERPRETER;
6233 }
6234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
6235 return rc;
6236}
6237
6238
6239/**
6240 * \#VMEXIT handler for MWAIT (SVM_EXIT_MWAIT). Conditional \#VMEXIT.
6241 */
6242HMSVM_EXIT_DECL hmR0SvmExitMwait(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6243{
6244 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6245 VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
6246 int rc = VBOXSTRICTRC_VAL(rc2);
6247 if ( rc == VINF_EM_HALT
6248 || rc == VINF_SUCCESS)
6249 {
6250 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3);
6251
6252 if ( rc == VINF_EM_HALT
6253 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
6254 {
6255 rc = VINF_SUCCESS;
6256 }
6257 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6258 }
6259 else
6260 {
6261 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0SvmExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
6262 rc = VERR_EM_INTERPRETER;
6263 }
6264 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
6265 ("hmR0SvmExitMwait: EMInterpretMWait failed rc=%Rrc\n", rc));
6266 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
6267 return rc;
6268}
6269
6270
6271/**
6272 * \#VMEXIT handler for shutdown (triple-fault) (SVM_EXIT_SHUTDOWN). Conditional
6273 * \#VMEXIT.
6274 */
6275HMSVM_EXIT_DECL hmR0SvmExitShutdown(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6276{
6277 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6278 return VINF_EM_RESET;
6279}
6280
6281
6282/**
6283 * \#VMEXIT handler for unexpected exits. Conditional \#VMEXIT.
6284 */
6285HMSVM_EXIT_DECL hmR0SvmExitUnexpected(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6286{
6287 RT_NOREF(pCtx);
6288 AssertMsgFailed(("hmR0SvmExitUnexpected: ExitCode=%#RX64\n", pSvmTransient->u64ExitCode));
6289 pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6290 return VERR_SVM_UNEXPECTED_EXIT;
6291}
6292
6293
6294/**
6295 * \#VMEXIT handler for CRx reads (SVM_EXIT_READ_CR*). Conditional \#VMEXIT.
6296 */
6297HMSVM_EXIT_DECL hmR0SvmExitReadCRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6298{
6299 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6300
6301 Log4(("hmR0SvmExitReadCRx: CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6302 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0]);
6303
6304 bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6305 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6306 if ( fSupportsDecodeAssists
6307 && fSupportsNextRipSave)
6308 {
6309 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6310 bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6311 if (fMovCRx)
6312 {
6313 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6314 uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0;
6315 uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6316 VBOXSTRICTRC rcStrict = IEMExecDecodedMovCRxRead(pVCpu, cbInstr, iGReg, iCrReg);
6317 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6318 return VBOXSTRICTRC_VAL(rcStrict);
6319 }
6320 /* else: SMSW instruction, fall back below to IEM for this. */
6321 }
6322
6323 VBOXSTRICTRC rc2 = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
6324 int rc = VBOXSTRICTRC_VAL(rc2);
6325 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3,
6326 ("hmR0SvmExitReadCRx: EMInterpretInstruction failed rc=%Rrc\n", rc));
6327 Assert((pSvmTransient->u64ExitCode - SVM_EXIT_READ_CR0) <= 15);
6328 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6329 return rc;
6330}
6331
6332
6333/**
6334 * \#VMEXIT handler for CRx writes (SVM_EXIT_WRITE_CR*). Conditional \#VMEXIT.
6335 */
6336HMSVM_EXIT_DECL hmR0SvmExitWriteCRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6337{
6338 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6339
6340 uint8_t const iCrReg = pSvmTransient->u64ExitCode - SVM_EXIT_WRITE_CR0;
6341 Assert(iCrReg <= 15);
6342
6343 VBOXSTRICTRC rcStrict = VERR_SVM_IPE_5;
6344 bool fDecodedInstr = false;
6345 bool const fSupportsDecodeAssists = hmR0SvmSupportsDecodeAssists(pVCpu, pCtx);
6346 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6347 if ( fSupportsDecodeAssists
6348 && fSupportsNextRipSave)
6349 {
6350 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6351 bool const fMovCRx = RT_BOOL(pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_MASK);
6352 if (fMovCRx)
6353 {
6354 uint8_t const cbInstr = pVmcb->ctrl.u64NextRIP - pCtx->rip;
6355 uint8_t const iGReg = pVmcb->ctrl.u64ExitInfo1 & SVM_EXIT1_MOV_CRX_GPR_NUMBER;
6356 Log4(("hmR0SvmExitWriteCRx: Mov CR%u w/ iGReg=%#x\n", iCrReg, iGReg));
6357 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, cbInstr, iCrReg, iGReg);
6358 fDecodedInstr = true;
6359 }
6360 /* else: LMSW or CLTS instruction, fall back below to IEM for this. */
6361 }
6362
6363 if (!fDecodedInstr)
6364 {
6365 Log4(("hmR0SvmExitWriteCRx: iCrReg=%#x\n", iCrReg));
6366 rcStrict = IEMExecOneBypassEx(pVCpu, CPUMCTX2CORE(pCtx), NULL);
6367 if (RT_UNLIKELY( rcStrict == VERR_IEM_ASPECT_NOT_IMPLEMENTED
6368 || rcStrict == VERR_IEM_INSTR_NOT_IMPLEMENTED))
6369 rcStrict = VERR_EM_INTERPRETER;
6370 }
6371
6372 if (rcStrict == VINF_SUCCESS)
6373 {
6374 switch (iCrReg)
6375 {
6376 case 0: /* CR0. */
6377 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6378 break;
6379
6380 case 3: /* CR3. */
6381 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
6382 break;
6383
6384 case 4: /* CR4. */
6385 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
6386 break;
6387
6388 case 8: /* CR8 (TPR). */
6389 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
6390 break;
6391
6392 default:
6393 AssertMsgFailed(("hmR0SvmExitWriteCRx: Invalid/Unexpected Write-CRx exit. u64ExitCode=%#RX64 %#x\n",
6394 pSvmTransient->u64ExitCode, iCrReg));
6395 break;
6396 }
6397 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6398 }
6399 else
6400 Assert(rcStrict == VERR_EM_INTERPRETER || rcStrict == VINF_PGM_CHANGE_MODE || rcStrict == VINF_PGM_SYNC_CR3);
6401 return VBOXSTRICTRC_TODO(rcStrict);
6402}
6403
6404
6405/**
6406 * \#VMEXIT handler for MSR read and writes (SVM_EXIT_MSR). Conditional
6407 * \#VMEXIT.
6408 */
6409HMSVM_EXIT_DECL hmR0SvmExitMsr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6410{
6411 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6412 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6413 PVM pVM = pVCpu->CTX_SUFF(pVM);
6414
6415 int rc;
6416 if (pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_WRITE)
6417 {
6418 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
6419 Log4(("MSR Write: idMsr=%#RX32\n", pCtx->ecx));
6420
6421 /* Handle TPR patching; intercepted LSTAR write. */
6422 if ( pVM->hm.s.fTPRPatchingActive
6423 && pCtx->ecx == MSR_K8_LSTAR)
6424 {
6425 if ((pCtx->eax & 0xff) != pSvmTransient->u8GuestTpr)
6426 {
6427 /* Our patch code uses LSTAR for TPR caching for 32-bit guests. */
6428 int rc2 = APICSetTpr(pVCpu, pCtx->eax & 0xff);
6429 AssertRC(rc2);
6430 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
6431 }
6432 rc = VINF_SUCCESS;
6433 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 2);
6434 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6435 return rc;
6436 }
6437
6438 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6439 if (fSupportsNextRipSave)
6440 {
6441 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6442 if (RT_LIKELY(rc == VINF_SUCCESS))
6443 {
6444 pCtx->rip = pVmcb->ctrl.u64NextRIP;
6445 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6446 }
6447 else
6448 AssertMsg( rc == VERR_EM_INTERPRETER
6449 || rc == VINF_CPUM_R3_MSR_WRITE, ("hmR0SvmExitMsr: EMInterpretWrmsr failed rc=%Rrc\n", rc));
6450 }
6451 else
6452 {
6453 rc = VBOXSTRICTRC_TODO(EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */));
6454 if (RT_LIKELY(rc == VINF_SUCCESS))
6455 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc); /* RIP updated by EMInterpretInstruction(). */
6456 else
6457 AssertMsg( rc == VERR_EM_INTERPRETER
6458 || rc == VINF_CPUM_R3_MSR_WRITE, ("hmR0SvmExitMsr: WrMsr. EMInterpretInstruction failed rc=%Rrc\n", rc));
6459 }
6460
6461 if (rc == VINF_SUCCESS)
6462 {
6463 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
6464 if ( pCtx->ecx >= MSR_IA32_X2APIC_START
6465 && pCtx->ecx <= MSR_IA32_X2APIC_END)
6466 {
6467 /*
6468 * We've already saved the APIC related guest-state (TPR) in hmR0SvmPostRunGuest(). When full APIC register
6469 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCB before
6470 * EMInterpretWrmsr() changes it.
6471 */
6472 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
6473 }
6474 else if (pCtx->ecx == MSR_K6_EFER)
6475 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
6476 else if (pCtx->ecx == MSR_IA32_TSC)
6477 pSvmTransient->fUpdateTscOffsetting = true;
6478 }
6479 }
6480 else
6481 {
6482 /* MSR Read access. */
6483 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
6484 Assert(pVmcb->ctrl.u64ExitInfo1 == SVM_EXIT1_MSR_READ);
6485 Log4(("MSR Read: idMsr=%#RX32\n", pCtx->ecx));
6486
6487 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6488 if (fSupportsNextRipSave)
6489 {
6490 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pCtx));
6491 if (RT_LIKELY(rc == VINF_SUCCESS))
6492 {
6493 pCtx->rip = pVmcb->ctrl.u64NextRIP;
6494 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6495 }
6496 else
6497 AssertMsg( rc == VERR_EM_INTERPRETER
6498 || rc == VINF_CPUM_R3_MSR_READ, ("hmR0SvmExitMsr: EMInterpretRdmsr failed rc=%Rrc\n", rc));
6499 }
6500 else
6501 {
6502 rc = VBOXSTRICTRC_TODO(EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0));
6503 if (RT_UNLIKELY(rc != VINF_SUCCESS))
6504 {
6505 AssertMsg( rc == VERR_EM_INTERPRETER
6506 || rc == VINF_CPUM_R3_MSR_READ, ("hmR0SvmExitMsr: RdMsr. EMInterpretInstruction failed rc=%Rrc\n", rc));
6507 }
6508 /* RIP updated by EMInterpretInstruction(). */
6509 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6510 }
6511 }
6512
6513 /* RIP has been updated by EMInterpret[Rd|Wr]msr() or EMInterpretInstruction(). */
6514 return rc;
6515}
6516
6517
6518/**
6519 * \#VMEXIT handler for DRx read (SVM_EXIT_READ_DRx). Conditional \#VMEXIT.
6520 */
6521HMSVM_EXIT_DECL hmR0SvmExitReadDRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6522{
6523 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6524 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
6525
6526 /** @todo Stepping with nested-guest. */
6527 if (!CPUMIsGuestInSvmNestedHwVirtMode(pCtx))
6528 {
6529 /* We should -not- get this #VMEXIT if the guest's debug registers were active. */
6530 if (pSvmTransient->fWasGuestDebugStateActive)
6531 {
6532 AssertMsgFailed(("hmR0SvmExitReadDRx: Unexpected exit %#RX32\n", (uint32_t)pSvmTransient->u64ExitCode));
6533 pVCpu->hm.s.u32HMError = (uint32_t)pSvmTransient->u64ExitCode;
6534 return VERR_SVM_UNEXPECTED_EXIT;
6535 }
6536
6537 /*
6538 * Lazy DR0-3 loading.
6539 */
6540 if (!pSvmTransient->fWasHyperDebugStateActive)
6541 {
6542 Assert(!DBGFIsStepping(pVCpu)); Assert(!pVCpu->hm.s.fSingleInstruction);
6543 Log5(("hmR0SvmExitReadDRx: Lazy loading guest debug registers\n"));
6544
6545 /* Don't intercept DRx read and writes. */
6546 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
6547 pVmcb->ctrl.u16InterceptRdDRx = 0;
6548 pVmcb->ctrl.u16InterceptWrDRx = 0;
6549 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_INTERCEPTS;
6550
6551 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
6552 VMMRZCallRing3Disable(pVCpu);
6553 HM_DISABLE_PREEMPT();
6554
6555 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
6556 CPUMR0LoadGuestDebugState(pVCpu, false /* include DR6 */);
6557 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
6558
6559 HM_RESTORE_PREEMPT();
6560 VMMRZCallRing3Enable(pVCpu);
6561
6562 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
6563 return VINF_SUCCESS;
6564 }
6565 }
6566
6567 /*
6568 * Interpret the read/writing of DRx.
6569 */
6570 /** @todo Decode assist. */
6571 VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0 /* pvFault */);
6572 Log5(("hmR0SvmExitReadDRx: Emulated DRx access: rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
6573 if (RT_LIKELY(rc == VINF_SUCCESS))
6574 {
6575 /* Not necessary for read accesses but whatever doesn't hurt for now, will be fixed with decode assist. */
6576 /** @todo CPUM should set this flag! */
6577 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6578 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
6579 }
6580 else
6581 Assert(rc == VERR_EM_INTERPRETER);
6582 return VBOXSTRICTRC_TODO(rc);
6583}
6584
6585
6586/**
6587 * \#VMEXIT handler for DRx write (SVM_EXIT_WRITE_DRx). Conditional \#VMEXIT.
6588 */
6589HMSVM_EXIT_DECL hmR0SvmExitWriteDRx(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6590{
6591 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6592 /* For now it's the same since we interpret the instruction anyway. Will change when using of Decode Assist is implemented. */
6593 int rc = hmR0SvmExitReadDRx(pVCpu, pCtx, pSvmTransient);
6594 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
6595 STAM_COUNTER_DEC(&pVCpu->hm.s.StatExitDRxRead);
6596 return rc;
6597}
6598
6599
6600/**
6601 * \#VMEXIT handler for XCRx write (SVM_EXIT_XSETBV). Conditional \#VMEXIT.
6602 */
6603HMSVM_EXIT_DECL hmR0SvmExitXsetbv(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6604{
6605 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6606
6607 /** @todo decode assists... */
6608 VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
6609 if (rcStrict == VINF_IEM_RAISED_XCPT)
6610 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6611
6612 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
6613 Log4(("hmR0SvmExitXsetbv: New XCR0=%#RX64 fLoadSaveGuestXcr0=%d (cr4=%RX64) rcStrict=%Rrc\n",
6614 pCtx->aXcr[0], pVCpu->hm.s.fLoadSaveGuestXcr0, pCtx->cr4, VBOXSTRICTRC_VAL(rcStrict)));
6615
6616 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6617 return VBOXSTRICTRC_TODO(rcStrict);
6618}
6619
6620
6621/**
6622 * \#VMEXIT handler for I/O instructions (SVM_EXIT_IOIO). Conditional \#VMEXIT.
6623 */
6624HMSVM_EXIT_DECL hmR0SvmExitIOInstr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6625{
6626 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6627
6628 /* I/O operation lookup arrays. */
6629 static uint32_t const s_aIOSize[8] = { 0, 1, 2, 0, 4, 0, 0, 0 }; /* Size of the I/O accesses in bytes. */
6630 static uint32_t const s_aIOOpAnd[8] = { 0, 0xff, 0xffff, 0, 0xffffffff, 0, 0, 0 }; /* AND masks for saving
6631 the result (in AL/AX/EAX). */
6632 Log4(("hmR0SvmExitIOInstr: CS:RIP=%04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
6633
6634 PVM pVM = pVCpu->CTX_SUFF(pVM);
6635 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6636
6637 /* Refer AMD spec. 15.10.2 "IN and OUT Behaviour" and Figure 15-2. "EXITINFO1 for IOIO Intercept" for the format. */
6638 SVMIOIOEXITINFO IoExitInfo;
6639 IoExitInfo.u = (uint32_t)pVmcb->ctrl.u64ExitInfo1;
6640 uint32_t uIOWidth = (IoExitInfo.u >> 4) & 0x7;
6641 uint32_t cbValue = s_aIOSize[uIOWidth];
6642 uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
6643
6644 if (RT_UNLIKELY(!cbValue))
6645 {
6646 AssertMsgFailed(("hmR0SvmExitIOInstr: Invalid IO operation. uIOWidth=%u\n", uIOWidth));
6647 return VERR_EM_INTERPRETER;
6648 }
6649
6650 VBOXSTRICTRC rcStrict;
6651 bool fUpdateRipAlready = false;
6652 if (IoExitInfo.n.u1STR)
6653 {
6654#ifdef VBOX_WITH_2ND_IEM_STEP
6655 /* INS/OUTS - I/O String instruction. */
6656 /** @todo Huh? why can't we use the segment prefix information given by AMD-V
6657 * in EXITINFO1? Investigate once this thing is up and running. */
6658 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, IoExitInfo.n.u16Port, cbValue,
6659 IoExitInfo.n.u1Type == SVM_IOIO_WRITE ? 'w' : 'r'));
6660 AssertReturn(pCtx->dx == IoExitInfo.n.u16Port, VERR_SVM_IPE_2);
6661 static IEMMODE const s_aenmAddrMode[8] =
6662 {
6663 (IEMMODE)-1, IEMMODE_16BIT, IEMMODE_32BIT, (IEMMODE)-1, IEMMODE_64BIT, (IEMMODE)-1, (IEMMODE)-1, (IEMMODE)-1
6664 };
6665 IEMMODE enmAddrMode = s_aenmAddrMode[(IoExitInfo.u >> 7) & 0x7];
6666 if (enmAddrMode != (IEMMODE)-1)
6667 {
6668 uint64_t cbInstr = pVmcb->ctrl.u64ExitInfo2 - pCtx->rip;
6669 if (cbInstr <= 15 && cbInstr >= 1)
6670 {
6671 Assert(cbInstr >= 1U + IoExitInfo.n.u1REP);
6672 if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6673 {
6674 /* Don't know exactly how to detect whether u3SEG is valid, currently
6675 only enabling it for Bulldozer and later with NRIP. OS/2 broke on
6676 2384 Opterons when only checking NRIP. */
6677 bool const fSupportsNextRipSave = hmR0SvmSupportsNextRipSave(pVCpu, pCtx);
6678 if ( fSupportsNextRipSave
6679 && pVM->cpum.ro.GuestFeatures.enmMicroarch >= kCpumMicroarch_AMD_15h_First)
6680 {
6681 AssertMsg(IoExitInfo.n.u3SEG == X86_SREG_DS || cbInstr > 1U + IoExitInfo.n.u1REP,
6682 ("u32Seg=%d cbInstr=%d u1REP=%d", IoExitInfo.n.u3SEG, cbInstr, IoExitInfo.n.u1REP));
6683 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1REP, (uint8_t)cbInstr,
6684 IoExitInfo.n.u3SEG, true /*fIoChecked*/);
6685 }
6686 else if (cbInstr == 1U + IoExitInfo.n.u1REP)
6687 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1REP, (uint8_t)cbInstr,
6688 X86_SREG_DS, true /*fIoChecked*/);
6689 else
6690 rcStrict = IEMExecOne(pVCpu);
6691 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
6692 }
6693 else
6694 {
6695 AssertMsg(IoExitInfo.n.u3SEG == X86_SREG_ES /*=0*/, ("%#x\n", IoExitInfo.n.u3SEG));
6696 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, IoExitInfo.n.u1REP, (uint8_t)cbInstr,
6697 true /*fIoChecked*/);
6698 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
6699 }
6700 }
6701 else
6702 {
6703 AssertMsgFailed(("rip=%RX64 nrip=%#RX64 cbInstr=%#RX64\n", pCtx->rip, pVmcb->ctrl.u64ExitInfo2, cbInstr));
6704 rcStrict = IEMExecOne(pVCpu);
6705 }
6706 }
6707 else
6708 {
6709 AssertMsgFailed(("IoExitInfo=%RX64\n", IoExitInfo.u));
6710 rcStrict = IEMExecOne(pVCpu);
6711 }
6712 fUpdateRipAlready = true;
6713
6714#else
6715 /* INS/OUTS - I/O String instruction. */
6716 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
6717
6718 /** @todo Huh? why can't we use the segment prefix information given by AMD-V
6719 * in EXITINFO1? Investigate once this thing is up and running. */
6720
6721 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
6722 if (rcStrict == VINF_SUCCESS)
6723 {
6724 if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6725 {
6726 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->fPrefix,
6727 (DISCPUMODE)pDis->uAddrMode, cbValue);
6728 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
6729 }
6730 else
6731 {
6732 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), IoExitInfo.n.u16Port, pDis->fPrefix,
6733 (DISCPUMODE)pDis->uAddrMode, cbValue);
6734 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
6735 }
6736 }
6737 else
6738 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
6739#endif
6740 }
6741 else
6742 {
6743 /* IN/OUT - I/O instruction. */
6744 Assert(!IoExitInfo.n.u1REP);
6745
6746 if (IoExitInfo.n.u1Type == SVM_IOIO_WRITE)
6747 {
6748 rcStrict = IOMIOPortWrite(pVM, pVCpu, IoExitInfo.n.u16Port, pCtx->eax & uAndVal, cbValue);
6749 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
6750 }
6751 else
6752 {
6753 uint32_t u32Val = 0;
6754 rcStrict = IOMIOPortRead(pVM, pVCpu, IoExitInfo.n.u16Port, &u32Val, cbValue);
6755 if (IOM_SUCCESS(rcStrict))
6756 {
6757 /* Save result of I/O IN instr. in AL/AX/EAX. */
6758 /** @todo r=bird: 32-bit op size should clear high bits of rax! */
6759 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
6760 }
6761 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
6762 HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pVmcb->ctrl.u64ExitInfo2, IoExitInfo.n.u16Port, uAndVal, cbValue);
6763
6764 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
6765 }
6766 }
6767
6768 if (IOM_SUCCESS(rcStrict))
6769 {
6770 /* AMD-V saves the RIP of the instruction following the IO instruction in EXITINFO2. */
6771 if (!fUpdateRipAlready)
6772 pCtx->rip = pVmcb->ctrl.u64ExitInfo2;
6773
6774 /*
6775 * If any I/O breakpoints are armed, we need to check if one triggered
6776 * and take appropriate action.
6777 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
6778 */
6779 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
6780 * execution engines about whether hyper BPs and such are pending. */
6781 uint32_t const uDr7 = pCtx->dr[7];
6782 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
6783 && X86_DR7_ANY_RW_IO(uDr7)
6784 && (pCtx->cr4 & X86_CR4_DE))
6785 || DBGFBpIsHwIoArmed(pVM)))
6786 {
6787 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
6788 VMMRZCallRing3Disable(pVCpu);
6789 HM_DISABLE_PREEMPT();
6790
6791 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
6792 CPUMR0DebugStateMaybeSaveGuest(pVCpu, false /*fDr6*/);
6793
6794 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, IoExitInfo.n.u16Port, cbValue);
6795 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
6796 {
6797 /* Raise #DB. */
6798 pVmcb->guest.u64DR6 = pCtx->dr[6];
6799 pVmcb->guest.u64DR7 = pCtx->dr[7];
6800 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
6801 hmR0SvmSetPendingXcptDB(pVCpu);
6802 }
6803 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
6804 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
6805 else if ( rcStrict2 != VINF_SUCCESS
6806 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
6807 rcStrict = rcStrict2;
6808 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
6809
6810 HM_RESTORE_PREEMPT();
6811 VMMRZCallRing3Enable(pVCpu);
6812 }
6813
6814 HMSVM_CHECK_SINGLE_STEP(pVCpu, rcStrict);
6815 }
6816
6817#ifdef VBOX_STRICT
6818 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
6819 Assert(IoExitInfo.n.u1Type == SVM_IOIO_READ);
6820 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
6821 Assert(IoExitInfo.n.u1Type == SVM_IOIO_WRITE);
6822 else
6823 {
6824 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
6825 * statuses, that the VMM device and some others may return. See
6826 * IOM_SUCCESS() for guidance. */
6827 AssertMsg( RT_FAILURE(rcStrict)
6828 || rcStrict == VINF_SUCCESS
6829 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
6830 || rcStrict == VINF_EM_DBG_BREAKPOINT
6831 || rcStrict == VINF_EM_RAW_GUEST_TRAP
6832 || rcStrict == VINF_EM_RAW_TO_R3
6833 || rcStrict == VINF_TRPM_XCPT_DISPATCHED
6834 || rcStrict == VINF_EM_TRIPLE_FAULT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
6835 }
6836#endif
6837 return VBOXSTRICTRC_TODO(rcStrict);
6838}
6839
6840
6841/**
6842 * \#VMEXIT handler for Nested Page-faults (SVM_EXIT_NPF). Conditional \#VMEXIT.
6843 */
6844HMSVM_EXIT_DECL hmR0SvmExitNestedPF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6845{
6846 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6847 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
6848
6849 PVM pVM = pVCpu->CTX_SUFF(pVM);
6850 Assert(pVM->hm.s.fNestedPaging);
6851
6852 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
6853
6854 /* See AMD spec. 15.25.6 "Nested versus Guest Page Faults, Fault Ordering" for VMCB details for #NPF. */
6855 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
6856 uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1;
6857 RTGCPHYS GCPhysFaultAddr = pVmcb->ctrl.u64ExitInfo2;
6858
6859 Log4(("#NPF at CS:RIP=%04x:%#RX64 faultaddr=%RGp errcode=%#x \n", pCtx->cs.Sel, pCtx->rip, GCPhysFaultAddr, u32ErrCode));
6860
6861#ifdef VBOX_HM_WITH_GUEST_PATCHING
6862 /* TPR patching for 32-bit guests, using the reserved bit in the page tables for MMIO regions. */
6863 if ( pVM->hm.s.fTprPatchingAllowed
6864 && (GCPhysFaultAddr & PAGE_OFFSET_MASK) == XAPIC_OFF_TPR
6865 && ( !(u32ErrCode & X86_TRAP_PF_P) /* Not present */
6866 || (u32ErrCode & (X86_TRAP_PF_P | X86_TRAP_PF_RSVD)) == (X86_TRAP_PF_P | X86_TRAP_PF_RSVD)) /* MMIO page. */
6867 && !CPUMIsGuestInLongModeEx(pCtx)
6868 && !CPUMGetGuestCPL(pVCpu)
6869 && pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
6870 {
6871 RTGCPHYS GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
6872 GCPhysApicBase &= PAGE_BASE_GC_MASK;
6873
6874 if (GCPhysFaultAddr == GCPhysApicBase + XAPIC_OFF_TPR)
6875 {
6876 /* Only attempt to patch the instruction once. */
6877 PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
6878 if (!pPatch)
6879 return VINF_EM_HM_PATCH_TPR_INSTR;
6880 }
6881 }
6882#endif
6883
6884 /*
6885 * Determine the nested paging mode.
6886 */
6887 PGMMODE enmNestedPagingMode;
6888#if HC_ARCH_BITS == 32
6889 if (CPUMIsGuestInLongModeEx(pCtx))
6890 enmNestedPagingMode = PGMMODE_AMD64_NX;
6891 else
6892#endif
6893 enmNestedPagingMode = PGMGetHostMode(pVM);
6894
6895 /*
6896 * MMIO optimization using the reserved (RSVD) bit in the guest page tables for MMIO pages.
6897 */
6898 int rc;
6899 Assert((u32ErrCode & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) != X86_TRAP_PF_RSVD);
6900 if ((u32ErrCode & (X86_TRAP_PF_RSVD | X86_TRAP_PF_P)) == (X86_TRAP_PF_RSVD | X86_TRAP_PF_P))
6901 {
6902 /* If event delivery causes an MMIO #NPF, go back to instruction emulation as
6903 otherwise injecting the original pending event would most likely cause the same MMIO #NPF. */
6904 if (pVCpu->hm.s.Event.fPending)
6905 return VINF_EM_RAW_INJECT_TRPM_EVENT;
6906
6907 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, enmNestedPagingMode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr,
6908 u32ErrCode);
6909 rc = VBOXSTRICTRC_VAL(rc2);
6910
6911 /*
6912 * If we succeed, resume guest execution.
6913 * If we fail in interpreting the instruction because we couldn't get the guest physical address
6914 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
6915 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
6916 * weird case. See @bugref{6043}.
6917 */
6918 if ( rc == VINF_SUCCESS
6919 || rc == VERR_PAGE_TABLE_NOT_PRESENT
6920 || rc == VERR_PAGE_NOT_PRESENT)
6921 {
6922 /* Successfully handled MMIO operation. */
6923 HMCPU_CF_SET(pVCpu, HM_CHANGED_SVM_GUEST_APIC_STATE);
6924 rc = VINF_SUCCESS;
6925 }
6926 return rc;
6927 }
6928
6929 TRPMAssertXcptPF(pVCpu, GCPhysFaultAddr, u32ErrCode);
6930 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, enmNestedPagingMode, u32ErrCode, CPUMCTX2CORE(pCtx), GCPhysFaultAddr);
6931 TRPMResetTrap(pVCpu);
6932
6933 Log4(("#NPF: PGMR0Trap0eHandlerNestedPaging returned %Rrc CS:RIP=%04x:%#RX64\n", rc, pCtx->cs.Sel, pCtx->rip));
6934
6935 /*
6936 * Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}.
6937 */
6938 if ( rc == VINF_SUCCESS
6939 || rc == VERR_PAGE_TABLE_NOT_PRESENT
6940 || rc == VERR_PAGE_NOT_PRESENT)
6941 {
6942 /* We've successfully synced our shadow page tables. */
6943 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
6944 rc = VINF_SUCCESS;
6945 }
6946
6947 return rc;
6948}
6949
6950
6951/**
6952 * \#VMEXIT handler for virtual interrupt (SVM_EXIT_VINTR). Conditional
6953 * \#VMEXIT.
6954 */
6955HMSVM_EXIT_DECL hmR0SvmExitVIntr(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6956{
6957 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6958 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
6959
6960 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
6961 hmR0SvmClearVirtIntrIntercept(pVmcb);
6962
6963 /* Deliver the pending interrupt/NMI via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
6964 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6965 return VINF_SUCCESS;
6966}
6967
6968
6969/**
6970 * \#VMEXIT handler for task switches (SVM_EXIT_TASK_SWITCH). Conditional
6971 * \#VMEXIT.
6972 */
6973HMSVM_EXIT_DECL hmR0SvmExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
6974{
6975 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
6976
6977 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
6978
6979#ifndef HMSVM_ALWAYS_TRAP_TASK_SWITCH
6980 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
6981#endif
6982
6983 /* Check if this task-switch occurred while delivering an event through the guest IDT. */
6984 if (pVCpu->hm.s.Event.fPending) /* Can happen with exceptions/NMI. See @bugref{8411}. */
6985 {
6986 /*
6987 * AMD-V provides us with the exception which caused the TS; we collect
6988 * the information in the call to hmR0SvmCheckExitDueToEventDelivery.
6989 */
6990 Log4(("hmR0SvmExitTaskSwitch: TS occurred during event delivery.\n"));
6991 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
6992 return VINF_EM_RAW_INJECT_TRPM_EVENT;
6993 }
6994
6995 /** @todo Emulate task switch someday, currently just going back to ring-3 for
6996 * emulation. */
6997 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
6998 return VERR_EM_INTERPRETER;
6999}
7000
7001
7002/**
7003 * \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
7004 */
7005HMSVM_EXIT_DECL hmR0SvmExitVmmCall(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7006{
7007 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
7009
7010 bool fRipUpdated;
7011 VBOXSTRICTRC rcStrict = HMSvmVmmcall(pVCpu, pCtx, &fRipUpdated);
7012 if (RT_SUCCESS(rcStrict))
7013 {
7014 /* Only update the RIP if we're continuing guest execution and not
7015 in the case of say VINF_GIM_R3_HYPERCALL. */
7016 if ( rcStrict == VINF_SUCCESS
7017 && !fRipUpdated)
7018 {
7019 hmR0SvmAdvanceRipHwAssist(pVCpu, pCtx, 3 /* cbInstr */);
7020 }
7021
7022 /* If the hypercall or TPR patching changes anything other than guest's general-purpose registers,
7023 we would need to reload the guest changed bits here before VM-entry. */
7024 return VBOXSTRICTRC_VAL(rcStrict);
7025 }
7026
7027 hmR0SvmSetPendingXcptUD(pVCpu);
7028 return VINF_SUCCESS;
7029}
7030
7031
7032/**
7033 * \#VMEXIT handler for VMMCALL (SVM_EXIT_VMMCALL). Conditional \#VMEXIT.
7034 */
7035HMSVM_EXIT_DECL hmR0SvmExitPause(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7036{
7037 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7038 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7039 return VINF_EM_RAW_INTERRUPT;
7040}
7041
7042
7043/**
7044 * \#VMEXIT handler for IRET (SVM_EXIT_IRET). Conditional \#VMEXIT.
7045 */
7046HMSVM_EXIT_DECL hmR0SvmExitIret(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7047{
7048 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7049
7050 /* Clear NMI blocking. */
7051 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7052
7053 /* Indicate that we no longer need to #VMEXIT when the guest is ready to receive NMIs, it is now ready. */
7054 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7055 hmR0SvmClearIretIntercept(pVmcb);
7056
7057 /* Deliver the pending NMI via hmR0SvmEvaluatePendingEvent() and resume guest execution. */
7058 return VINF_SUCCESS;
7059}
7060
7061
7062/**
7063 * \#VMEXIT handler for page-fault exceptions (SVM_EXIT_EXCEPTION_14).
7064 * Conditional \#VMEXIT.
7065 */
7066HMSVM_EXIT_DECL hmR0SvmExitXcptPF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7067{
7068 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7069 HMSVM_ASSERT_NOT_IN_NESTED_GUEST(pCtx);
7070
7071 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7072
7073 /* See AMD spec. 15.12.15 "#PF (Page Fault)". */
7074 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7075 uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1;
7076 RTGCUINTPTR uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
7077 PVM pVM = pVCpu->CTX_SUFF(pVM);
7078
7079#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) || defined(HMSVM_ALWAYS_TRAP_PF)
7080 if (pVM->hm.s.fNestedPaging)
7081 {
7082 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7083 if (!pSvmTransient->fVectoringDoublePF)
7084 {
7085 /* A genuine guest #PF, reflect it to the guest. */
7086 hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
7087 Log4(("#PF: Guest page fault at %04X:%RGv FaultAddr=%RGv ErrCode=%#x\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip,
7088 uFaultAddress, u32ErrCode));
7089 }
7090 else
7091 {
7092 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7093 hmR0SvmSetPendingXcptDF(pVCpu);
7094 Log4(("Pending #DF due to vectoring #PF. NP\n"));
7095 }
7096 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7097 return VINF_SUCCESS;
7098 }
7099#endif
7100
7101 Assert(!pVM->hm.s.fNestedPaging);
7102
7103#ifdef VBOX_HM_WITH_GUEST_PATCHING
7104 /* Shortcut for APIC TPR reads and writes; only applicable to 32-bit guests. */
7105 if ( pVM->hm.s.fTprPatchingAllowed
7106 && (uFaultAddress & 0xfff) == XAPIC_OFF_TPR
7107 && !(u32ErrCode & X86_TRAP_PF_P) /* Not present. */
7108 && !CPUMIsGuestInLongModeEx(pCtx)
7109 && !CPUMGetGuestCPL(pVCpu)
7110 && pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
7111 {
7112 RTGCPHYS GCPhysApicBase;
7113 GCPhysApicBase = APICGetBaseMsrNoCheck(pVCpu);
7114 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7115
7116 /* Check if the page at the fault-address is the APIC base. */
7117 RTGCPHYS GCPhysPage;
7118 int rc2 = PGMGstGetPage(pVCpu, (RTGCPTR)uFaultAddress, NULL /* pfFlags */, &GCPhysPage);
7119 if ( rc2 == VINF_SUCCESS
7120 && GCPhysPage == GCPhysApicBase)
7121 {
7122 /* Only attempt to patch the instruction once. */
7123 PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
7124 if (!pPatch)
7125 return VINF_EM_HM_PATCH_TPR_INSTR;
7126 }
7127 }
7128#endif
7129
7130 Log4(("#PF: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 u32ErrCode %#RX32 cr3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
7131 pCtx->rip, u32ErrCode, pCtx->cr3));
7132
7133 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
7134 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
7135 if (pSvmTransient->fVectoringPF)
7136 {
7137 Assert(pVCpu->hm.s.Event.fPending);
7138 return VINF_EM_RAW_INJECT_TRPM_EVENT;
7139 }
7140
7141 TRPMAssertXcptPF(pVCpu, uFaultAddress, u32ErrCode);
7142 int rc = PGMTrap0eHandler(pVCpu, u32ErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
7143
7144 Log4(("#PF rc=%Rrc\n", rc));
7145
7146 if (rc == VINF_SUCCESS)
7147 {
7148 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
7149 TRPMResetTrap(pVCpu);
7150 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7151 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7152 return rc;
7153 }
7154 else if (rc == VINF_EM_RAW_GUEST_TRAP)
7155 {
7156 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7157
7158 if (!pSvmTransient->fVectoringDoublePF)
7159 {
7160 /* It's a guest page fault and needs to be reflected to the guest. */
7161 u32ErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
7162 TRPMResetTrap(pVCpu);
7163 hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
7164 }
7165 else
7166 {
7167 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7168 TRPMResetTrap(pVCpu);
7169 hmR0SvmSetPendingXcptDF(pVCpu);
7170 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
7171 }
7172
7173 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7174 return VINF_SUCCESS;
7175 }
7176
7177 TRPMResetTrap(pVCpu);
7178 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
7179 return rc;
7180}
7181
7182
7183/**
7184 * \#VMEXIT handler for device-not-available exceptions (SVM_EXIT_EXCEPTION_7).
7185 * Conditional \#VMEXIT.
7186 */
7187HMSVM_EXIT_DECL hmR0SvmExitXcptNM(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7188{
7189 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7190
7191 /* Paranoia; Ensure we cannot be called as a result of event delivery. */
7192 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7193 Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7194
7195 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
7196 VMMRZCallRing3Disable(pVCpu);
7197 HM_DISABLE_PREEMPT();
7198
7199 int rc;
7200 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
7201 if (pSvmTransient->fWasGuestFPUStateActive)
7202 {
7203 rc = VINF_EM_RAW_GUEST_TRAP;
7204 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
7205 }
7206 else
7207 {
7208#ifndef HMSVM_ALWAYS_TRAP_ALL_XCPTS
7209 Assert(!pSvmTransient->fWasGuestFPUStateActive);
7210#endif
7211 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu); /* (No need to set HM_CHANGED_HOST_CONTEXT for SVM.) */
7212 Assert( rc == VINF_EM_RAW_GUEST_TRAP
7213 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
7214 }
7215
7216 HM_RESTORE_PREEMPT();
7217 VMMRZCallRing3Enable(pVCpu);
7218
7219 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
7220 {
7221 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
7222 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7223 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
7224 pVCpu->hm.s.fPreloadGuestFpu = true;
7225 }
7226 else
7227 {
7228 /* Forward #NM to the guest. */
7229 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
7230 hmR0SvmSetPendingXcptNM(pVCpu);
7231 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
7232 }
7233 return VINF_SUCCESS;
7234}
7235
7236
7237/**
7238 * \#VMEXIT handler for undefined opcode (SVM_EXIT_EXCEPTION_6).
7239 * Conditional \#VMEXIT.
7240 */
7241HMSVM_EXIT_DECL hmR0SvmExitXcptUD(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7242{
7243 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7244
7245 /* Paranoia; Ensure we cannot be called as a result of event delivery. */
7246 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7247 Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7248
7249 int rc = VERR_SVM_UNEXPECTED_XCPT_EXIT;
7250 if (pVCpu->hm.s.fGIMTrapXcptUD)
7251 {
7252 uint8_t cbInstr = 0;
7253 VBOXSTRICTRC rcStrict = GIMXcptUD(pVCpu, pCtx, NULL /* pDis */, &cbInstr);
7254 if (rcStrict == VINF_SUCCESS)
7255 {
7256 /* #UD #VMEXIT does not have valid NRIP information, manually advance RIP. See @bugref{7270#c170}. */
7257 hmR0SvmAdvanceRipDumb(pVCpu, pCtx, cbInstr);
7258 rc = VINF_SUCCESS;
7259 HMSVM_CHECK_SINGLE_STEP(pVCpu, rc);
7260 }
7261 else if (rcStrict == VINF_GIM_HYPERCALL_CONTINUING)
7262 rc = VINF_SUCCESS;
7263 else if (rcStrict == VINF_GIM_R3_HYPERCALL)
7264 rc = VINF_GIM_R3_HYPERCALL;
7265 else
7266 Assert(RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
7267 }
7268
7269 /* If the GIM #UD exception handler didn't succeed for some reason or wasn't needed, raise #UD. */
7270 if (RT_FAILURE(rc))
7271 {
7272 hmR0SvmSetPendingXcptUD(pVCpu);
7273 rc = VINF_SUCCESS;
7274 }
7275
7276 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7277 return rc;
7278}
7279
7280
7281/**
7282 * \#VMEXIT handler for math-fault exceptions (SVM_EXIT_EXCEPTION_16).
7283 * Conditional \#VMEXIT.
7284 */
7285HMSVM_EXIT_DECL hmR0SvmExitXcptMF(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7286{
7287 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7288
7289 /* Paranoia; Ensure we cannot be called as a result of event delivery. */
7290 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7291 Assert(!pVmcb->ctrl.ExitIntInfo.n.u1Valid); NOREF(pVmcb);
7292
7293 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
7294
7295 if (!(pCtx->cr0 & X86_CR0_NE))
7296 {
7297 PVM pVM = pVCpu->CTX_SUFF(pVM);
7298 PDISSTATE pDis = &pVCpu->hm.s.DisState;
7299 unsigned cbOp;
7300 int rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
7301 if (RT_SUCCESS(rc))
7302 {
7303 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
7304 /** @todo FERR intercept when in nested-guest mode? */
7305 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
7306 if (RT_SUCCESS(rc))
7307 pCtx->rip += cbOp;
7308 }
7309 else
7310 Log4(("hmR0SvmExitXcptMF: EMInterpretDisasCurrent returned %Rrc uOpCode=%#x\n", rc, pDis->pCurInstr->uOpcode));
7311 return rc;
7312 }
7313
7314 hmR0SvmSetPendingXcptMF(pVCpu);
7315 return VINF_SUCCESS;
7316}
7317
7318
7319/**
7320 * \#VMEXIT handler for debug exceptions (SVM_EXIT_EXCEPTION_1). Conditional
7321 * \#VMEXIT.
7322 */
7323HMSVM_EXIT_DECL hmR0SvmExitXcptDB(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7324{
7325 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7326
7327 /* If this #DB is the result of delivering an event, go back to the interpreter. */
7328 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7329 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
7330 {
7331 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
7332 return VINF_EM_RAW_INJECT_TRPM_EVENT;
7333 }
7334
7335 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
7336
7337 /* This can be a fault-type #DB (instruction breakpoint) or a trap-type #DB (data breakpoint). However, for both cases
7338 DR6 and DR7 are updated to what the exception handler expects. See AMD spec. 15.12.2 "#DB (Debug)". */
7339 PVM pVM = pVCpu->CTX_SUFF(pVM);
7340 PSVMVMCB pVmcb = pVCpu->hm.s.svm.pVmcb;
7341 int rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), pVmcb->guest.u64DR6, pVCpu->hm.s.fSingleInstruction);
7342 if (rc == VINF_EM_RAW_GUEST_TRAP)
7343 {
7344 Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> guest trap\n", pVmcb->guest.u64DR6));
7345 if (CPUMIsHyperDebugStateActive(pVCpu))
7346 CPUMSetGuestDR6(pVCpu, CPUMGetGuestDR6(pVCpu) | pVmcb->guest.u64DR6);
7347
7348 /* Reflect the exception back to the guest. */
7349 hmR0SvmSetPendingXcptDB(pVCpu);
7350 rc = VINF_SUCCESS;
7351 }
7352
7353 /*
7354 * Update DR6.
7355 */
7356 if (CPUMIsHyperDebugStateActive(pVCpu))
7357 {
7358 Log5(("hmR0SvmExitXcptDB: DR6=%#RX64 -> %Rrc\n", pVmcb->guest.u64DR6, rc));
7359 pVmcb->guest.u64DR6 = X86_DR6_INIT_VAL;
7360 pVmcb->ctrl.u32VmcbCleanBits &= ~HMSVM_VMCB_CLEAN_DRX;
7361 }
7362 else
7363 {
7364 AssertMsg(rc == VINF_SUCCESS, ("rc=%Rrc\n", rc));
7365 Assert(!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu));
7366 }
7367
7368 return rc;
7369}
7370
7371
7372/**
7373 * \#VMEXIT handler for alignment check exceptions (SVM_EXIT_EXCEPTION_17).
7374 * Conditional \#VMEXIT.
7375 */
7376HMSVM_EXIT_DECL hmR0SvmExitXcptAC(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7377{
7378 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7379
7380 /** @todo if triple-fault is returned in nested-guest scenario convert to a
7381 * shutdown VMEXIT. */
7382 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7383
7384 SVMEVENT Event;
7385 Event.u = 0;
7386 Event.n.u1Valid = 1;
7387 Event.n.u3Type = SVM_EVENT_EXCEPTION;
7388 Event.n.u8Vector = X86_XCPT_AC;
7389 Event.n.u1ErrorCodeValid = 1;
7390 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7391 return VINF_SUCCESS;
7392}
7393
7394
7395/**
7396 * \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_EXCEPTION_3).
7397 * Conditional \#VMEXIT.
7398 */
7399HMSVM_EXIT_DECL hmR0SvmExitXcptBP(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7400{
7401 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7402
7403 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7404
7405 int rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
7406 if (rc == VINF_EM_RAW_GUEST_TRAP)
7407 {
7408 SVMEVENT Event;
7409 Event.u = 0;
7410 Event.n.u1Valid = 1;
7411 Event.n.u3Type = SVM_EVENT_EXCEPTION;
7412 Event.n.u8Vector = X86_XCPT_BP;
7413 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7414 }
7415
7416 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
7417 return rc;
7418}
7419
7420
7421#if defined(HMSVM_ALWAYS_TRAP_ALL_XCPTS) || defined(VBOX_WITH_NESTED_HWVIRT)
7422/**
7423 * \#VMEXIT handler for generic exceptions. Conditional \#VMEXIT.
7424 */
7425HMSVM_EXIT_DECL hmR0SvmExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7426{
7427 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7428
7429 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7430
7431 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7432 uint8_t const uVector = pVmcb->ctrl.u64ExitCode - SVM_EXIT_EXCEPTION_0;
7433 uint32_t const uErrCode = pVmcb->ctrl.u64ExitInfo1;
7434 Assert(pSvmTransient->u64ExitCode == pVmcb->ctrl.u64ExitCode);
7435 Assert(uVector <= X86_XCPT_LAST);
7436 Log4(("hmR0SvmExitXcptGeneric: uVector=%#x uErrCode=%u\n", uVector, uErrCode));
7437
7438
7439 SVMEVENT Event;
7440 Event.u = 0;
7441 Event.n.u1Valid = 1;
7442 Event.n.u3Type = SVM_EVENT_EXCEPTION;
7443 Event.n.u8Vector = uVector;
7444 switch (uVector)
7445 {
7446 /* Shouldn't be here for reflecting #PFs (among other things, the fault address isn't passed along). */
7447 case X86_XCPT_PF: AssertMsgFailed(("hmR0SvmExitXcptGeneric: Unexpected exception")); return VERR_SVM_IPE_5;
7448 case X86_XCPT_DF:
7449 case X86_XCPT_TS:
7450 case X86_XCPT_NP:
7451 case X86_XCPT_SS:
7452 case X86_XCPT_GP:
7453 case X86_XCPT_AC:
7454 {
7455 Event.n.u1ErrorCodeValid = 1;
7456 Event.n.u32ErrorCode = uErrCode;
7457 break;
7458 }
7459 }
7460
7461 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7462 return VINF_SUCCESS;
7463}
7464#endif
7465
7466#ifdef VBOX_WITH_NESTED_HWVIRT
7467/**
7468 * \#VMEXIT handler for #PF occuring while in nested-guest execution
7469 * (SVM_EXIT_EXCEPTION_14). Conditional \#VMEXIT.
7470 */
7471HMSVM_EXIT_DECL hmR0SvmExitXcptPFNested(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7472{
7473 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7474
7475 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7476
7477 /* See AMD spec. 15.12.15 "#PF (Page Fault)". */
7478 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7479 uint32_t u32ErrCode = pVmcb->ctrl.u64ExitInfo1;
7480 uint64_t const uFaultAddress = pVmcb->ctrl.u64ExitInfo2;
7481
7482 Log4(("#PFNested: uFaultAddress=%#RX64 CS:RIP=%#04x:%#RX64 u32ErrCode=%#RX32 CR3=%#RX64\n", uFaultAddress, pCtx->cs.Sel,
7483 pCtx->rip, u32ErrCode, pCtx->cr3));
7484
7485 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
7486 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
7487 if (pSvmTransient->fVectoringPF)
7488 {
7489 Assert(pVCpu->hm.s.Event.fPending);
7490 return VINF_EM_RAW_INJECT_TRPM_EVENT;
7491 }
7492
7493 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
7494
7495 TRPMAssertXcptPF(pVCpu, uFaultAddress, u32ErrCode);
7496 int rc = PGMTrap0eHandler(pVCpu, u32ErrCode, CPUMCTX2CORE(pCtx), (RTGCPTR)uFaultAddress);
7497
7498 Log4(("#PFNested: rc=%Rrc\n", rc));
7499
7500 if (rc == VINF_SUCCESS)
7501 {
7502 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
7503 TRPMResetTrap(pVCpu);
7504 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
7505 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7506 return rc;
7507 }
7508
7509 if (rc == VINF_EM_RAW_GUEST_TRAP)
7510 {
7511 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
7512
7513 if (!pSvmTransient->fVectoringDoublePF)
7514 {
7515 /* It's a nested-guest page fault and needs to be reflected to the nested-guest. */
7516 u32ErrCode = TRPMGetErrorCode(pVCpu); /* The error code might have been changed. */
7517 TRPMResetTrap(pVCpu);
7518 hmR0SvmSetPendingXcptPF(pVCpu, pCtx, u32ErrCode, uFaultAddress);
7519 }
7520 else
7521 {
7522 /* A nested-guest page-fault occurred during delivery of a page-fault. Inject #DF. */
7523 TRPMResetTrap(pVCpu);
7524 hmR0SvmSetPendingXcptDF(pVCpu);
7525 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
7526 }
7527
7528 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
7529 return VINF_SUCCESS;
7530 }
7531
7532 TRPMResetTrap(pVCpu);
7533 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
7534 return rc;
7535}
7536
7537
7538/**
7539 * \#VMEXIT handler for CLGI (SVM_EXIT_CLGI). Conditional \#VMEXIT.
7540 */
7541HMSVM_EXIT_DECL hmR0SvmExitClgi(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7542{
7543 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7544
7545 /** @todo Stat. */
7546 /* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClgi); */
7547 uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7548 VBOXSTRICTRC rcStrict = IEMExecDecodedClgi(pVCpu, cbInstr);
7549
7550 /*
7551 * The guest should no longer receive interrupts. Until VGIF is supported,
7552 * clear virtual interrupt intercepts here.
7553 */
7554 PSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7555 hmR0SvmClearVirtIntrIntercept(pVmcb);
7556
7557 return VBOXSTRICTRC_VAL(rcStrict);
7558}
7559
7560
7561/**
7562 * \#VMEXIT handler for STGI (SVM_EXIT_STGI). Conditional \#VMEXIT.
7563 */
7564HMSVM_EXIT_DECL hmR0SvmExitStgi(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7565{
7566 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7567
7568 /** @todo Stat. */
7569 /* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitStgi); */
7570 uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7571 VBOXSTRICTRC rcStrict = IEMExecDecodedStgi(pVCpu, cbInstr);
7572 return VBOXSTRICTRC_VAL(rcStrict);
7573}
7574
7575
7576/**
7577 * \#VMEXIT handler for VMLOAD (SVM_EXIT_VMLOAD). Conditional \#VMEXIT.
7578 */
7579HMSVM_EXIT_DECL hmR0SvmExitVmload(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7580{
7581 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7582
7583#ifdef VBOX_STRICT
7584 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7585 Assert(pVmcb);
7586 Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
7587 RT_NOREF(pVmcb);
7588#endif
7589
7590 /** @todo Stat. */
7591 /* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmload); */
7592 uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7593 VBOXSTRICTRC rcStrict = IEMExecDecodedVmload(pVCpu, cbInstr);
7594 if (rcStrict == VINF_SUCCESS)
7595 {
7596 /* We skip flagging changes made to LSTAR, STAR, SFMASK and other MSRs as they are always re-loaded. */
7597 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7598 | HM_CHANGED_GUEST_TR
7599 | HM_CHANGED_GUEST_LDTR);
7600 }
7601 return VBOXSTRICTRC_VAL(rcStrict);
7602}
7603
7604
7605/**
7606 * \#VMEXIT handler for VMSAVE (SVM_EXIT_VMSAVE). Conditional \#VMEXIT.
7607 */
7608HMSVM_EXIT_DECL hmR0SvmExitVmsave(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7609{
7610 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7611
7612#ifdef VBOX_STRICT
7613 PCSVMVMCB pVmcb = hmR0SvmGetCurrentVmcb(pVCpu, pCtx);
7614 Assert(pVmcb);
7615 Assert(!pVmcb->ctrl.LbrVirt.n.u1VirtVmsaveVmload);
7616 RT_NOREF(pVmcb);
7617#endif
7618
7619 /** @todo Stat. */
7620 /* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmsave); */
7621 uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7622 VBOXSTRICTRC rcStrict = IEMExecDecodedVmsave(pVCpu, cbInstr);
7623 return VBOXSTRICTRC_VAL(rcStrict);
7624}
7625
7626
7627/**
7628 * \#VMEXIT handler for INVLPGA (SVM_EXIT_INVLPGA). Conditional \#VMEXIT.
7629 */
7630HMSVM_EXIT_DECL hmR0SvmExitInvlpga(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7631{
7632 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7633 /** @todo Stat. */
7634 /* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpga); */
7635 uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7636 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpga(pVCpu, cbInstr);
7637 return VBOXSTRICTRC_VAL(rcStrict);
7638}
7639
7640
7641/**
7642 * \#VMEXIT handler for STGI (SVM_EXIT_VMRUN). Conditional \#VMEXIT.
7643 */
7644HMSVM_EXIT_DECL hmR0SvmExitVmrun(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7645{
7646 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7647 /** @todo Stat. */
7648 /* STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmrun); */
7649#if 0
7650 VBOXSTRICTRC rcStrict;
7651 uint8_t const cbInstr = hmR0SvmGetInstrLengthHwAssist(pVCpu, pCtx, 3);
7652 rcStrict = IEMExecDecodedVmrun(pVCpu, cbInstr);
7653 Log4(("IEMExecDecodedVmrun: returned %d\n", VBOXSTRICTRC_VAL(rcStrict)));
7654 if (rcStrict == VINF_SUCCESS)
7655 {
7656 rcStrict = VINF_SVM_VMRUN;
7657 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7658 }
7659 return VBOXSTRICTRC_VAL(rcStrict);
7660#endif
7661 return VERR_EM_INTERPRETER;
7662}
7663
7664
7665/**
7666 * Nested-guest \#VMEXIT handler for debug exceptions (SVM_EXIT_EXCEPTION_1).
7667 * Unconditional \#VMEXIT.
7668 */
7669HMSVM_EXIT_DECL hmR0SvmNestedExitXcptDB(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7670{
7671 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7672
7673 /* If this #DB is the result of delivering an event, go back to the interpreter. */
7674 /** @todo if triple-fault is returned in nested-guest scenario convert to a
7675 * shutdown VMEXIT. */
7676 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7677 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
7678 {
7679 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
7680 return VINF_EM_RAW_INJECT_TRPM_EVENT;
7681 }
7682
7683 hmR0SvmSetPendingXcptDB(pVCpu);
7684 return VINF_SUCCESS;
7685}
7686
7687
7688/**
7689 * Nested-guest \#VMEXIT handler for breakpoint exceptions (SVM_EXIT_EXCEPTION_3).
7690 * Conditional \#VMEXIT.
7691 */
7692HMSVM_EXIT_DECL hmR0SvmNestedExitXcptBP(PVMCPU pVCpu, PCPUMCTX pCtx, PSVMTRANSIENT pSvmTransient)
7693{
7694 HMSVM_VALIDATE_EXIT_HANDLER_PARAMS();
7695
7696 /** @todo if triple-fault is returned in nested-guest scenario convert to a
7697 * shutdown VMEXIT. */
7698 HMSVM_CHECK_EXIT_DUE_TO_EVENT_DELIVERY();
7699
7700 SVMEVENT Event;
7701 Event.u = 0;
7702 Event.n.u1Valid = 1;
7703 Event.n.u3Type = SVM_EVENT_EXCEPTION;
7704 Event.n.u8Vector = X86_XCPT_BP;
7705 hmR0SvmSetPendingEvent(pVCpu, &Event, 0 /* GCPtrFaultAddress */);
7706 return VINF_SUCCESS;
7707}
7708
7709#endif /* VBOX_WITH_NESTED_HWVIRT */
7710
7711
7712/** @} */
7713
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette