VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 54098

Last change on this file since 54098 was 54098, checked in by vboxsync, 10 years ago

VMM/HMVMXR0: spaces.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 506.1 KB
Line 
1/* $Id: HMVMXR0.cpp 54098 2015-02-06 14:41:41Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39# include "dtrace/VBoxVMM.h"
40
41#ifdef DEBUG_ramshankar
42# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
43# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
44# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_CHECK_GUEST_STATE
46# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
47# define HMVMX_ALWAYS_TRAP_PF
48# define HMVMX_ALWAYS_SWAP_FPU_STATE
49# define HMVMX_ALWAYS_FLUSH_TLB
50# define HMVMX_ALWAYS_SWAP_EFER
51#endif
52
53
54/*******************************************************************************
55* Defined Constants And Macros *
56*******************************************************************************/
57#if defined(RT_ARCH_AMD64)
58# define HMVMX_IS_64BIT_HOST_MODE() (true)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
61extern "C" uint32_t g_fVMXIs64bitHost;
62# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
63typedef uint64_t HMVMXHCUINTREG;
64#else
65# define HMVMX_IS_64BIT_HOST_MODE() (false)
66typedef RTHCUINTREG HMVMXHCUINTREG;
67#endif
68
69/** Use the function table. */
70#define HMVMX_USE_FUNCTION_TABLE
71
72/** Determine which tagged-TLB flush handler to use. */
73#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
74#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
75#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
76#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
77
78/** @name Updated-guest-state flags.
79 * @{ */
80#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
81#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
82#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
83#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
84#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
85#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
86#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
87#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
88#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
89#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
90#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
91#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
92#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
93#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
94#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
95#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
96#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
97#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
98#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
99#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
100#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
101 | HMVMX_UPDATED_GUEST_RSP \
102 | HMVMX_UPDATED_GUEST_RFLAGS \
103 | HMVMX_UPDATED_GUEST_CR0 \
104 | HMVMX_UPDATED_GUEST_CR3 \
105 | HMVMX_UPDATED_GUEST_CR4 \
106 | HMVMX_UPDATED_GUEST_GDTR \
107 | HMVMX_UPDATED_GUEST_IDTR \
108 | HMVMX_UPDATED_GUEST_LDTR \
109 | HMVMX_UPDATED_GUEST_TR \
110 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
111 | HMVMX_UPDATED_GUEST_DEBUG \
112 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
113 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
114 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
115 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
116 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
117 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
118 | HMVMX_UPDATED_GUEST_INTR_STATE \
119 | HMVMX_UPDATED_GUEST_APIC_STATE)
120/** @} */
121
122/** @name
123 * Flags to skip redundant reads of some common VMCS fields that are not part of
124 * the guest-CPU state but are in the transient structure.
125 */
126#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
127#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
131#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
132#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
133/** @} */
134
135/** @name
136 * States of the VMCS.
137 *
138 * This does not reflect all possible VMCS states but currently only those
139 * needed for maintaining the VMCS consistently even when thread-context hooks
140 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
141 */
142#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
143#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
144#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
145/** @} */
146
147/**
148 * Exception bitmap mask for real-mode guests (real-on-v86).
149 *
150 * We need to intercept all exceptions manually except:
151 * - #NM, #MF handled in hmR0VmxLoadSharedCR0().
152 * - #DB handled in hmR0VmxLoadSharedDebugState().
153 * - #PF need not be intercepted even in real-mode if we have Nested Paging
154 * support.
155 */
156#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
157 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
158 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
159 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
160 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
161 /* RT_BIT(X86_XCPT_MF) */ | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
162 | RT_BIT(X86_XCPT_XF))
163
164/**
165 * Exception bitmap mask for all contributory exceptions.
166 *
167 * Page fault is deliberately excluded here as it's conditional as to whether
168 * it's contributory or benign. Page faults are handled separately.
169 */
170#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
171 | RT_BIT(X86_XCPT_DE))
172
173/** Maximum VM-instruction error number. */
174#define HMVMX_INSTR_ERROR_MAX 28
175
176/** Profiling macro. */
177#ifdef HM_PROFILE_EXIT_DISPATCH
178# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
179# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
180#else
181# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
182# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
183#endif
184
185/** Assert that preemption is disabled or covered by thread-context hooks. */
186#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
187 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
188
189/** Assert that we haven't migrated CPUs when thread-context hooks are not
190 * used. */
191#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
192 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
193 ("Illegal migration! Entered on CPU %u Current %u\n", \
194 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
195
196/** Helper macro for VM-exit handlers called unexpectedly. */
197#define HMVMX_RETURN_UNEXPECTED_EXIT() \
198 do { \
199 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
200 return VERR_VMX_UNEXPECTED_EXIT; \
201 } while (0)
202
203
204/*******************************************************************************
205* Structures and Typedefs *
206*******************************************************************************/
207/**
208 * VMX transient state.
209 *
210 * A state structure for holding miscellaneous information across
211 * VMX non-root operation and restored after the transition.
212 */
213typedef struct VMXTRANSIENT
214{
215 /** The host's rflags/eflags. */
216 RTCCUINTREG uEflags;
217#if HC_ARCH_BITS == 32
218 uint32_t u32Alignment0;
219#endif
220 /** The guest's TPR value used for TPR shadowing. */
221 uint8_t u8GuestTpr;
222 /** Alignment. */
223 uint8_t abAlignment0[7];
224
225 /** The basic VM-exit reason. */
226 uint16_t uExitReason;
227 /** Alignment. */
228 uint16_t u16Alignment0;
229 /** The VM-exit interruption error code. */
230 uint32_t uExitIntErrorCode;
231 /** The VM-exit exit code qualification. */
232 uint64_t uExitQualification;
233
234 /** The VM-exit interruption-information field. */
235 uint32_t uExitIntInfo;
236 /** The VM-exit instruction-length field. */
237 uint32_t cbInstr;
238 /** The VM-exit instruction-information field. */
239 union
240 {
241 /** Plain unsigned int representation. */
242 uint32_t u;
243 /** INS and OUTS information. */
244 struct
245 {
246 uint32_t u6Reserved0 : 7;
247 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
248 uint32_t u3AddrSize : 3;
249 uint32_t u5Reserved1 : 5;
250 /** The segment register (X86_SREG_XXX). */
251 uint32_t iSegReg : 3;
252 uint32_t uReserved2 : 14;
253 } StrIo;
254 } ExitInstrInfo;
255 /** Whether the VM-entry failed or not. */
256 bool fVMEntryFailed;
257 /** Alignment. */
258 uint8_t abAlignment1[3];
259
260 /** The VM-entry interruption-information field. */
261 uint32_t uEntryIntInfo;
262 /** The VM-entry exception error code field. */
263 uint32_t uEntryXcptErrorCode;
264 /** The VM-entry instruction length field. */
265 uint32_t cbEntryInstr;
266
267 /** IDT-vectoring information field. */
268 uint32_t uIdtVectoringInfo;
269 /** IDT-vectoring error code. */
270 uint32_t uIdtVectoringErrorCode;
271
272 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
273 uint32_t fVmcsFieldsRead;
274
275 /** Whether the guest FPU was active at the time of VM-exit. */
276 bool fWasGuestFPUStateActive;
277 /** Whether the guest debug state was active at the time of VM-exit. */
278 bool fWasGuestDebugStateActive;
279 /** Whether the hyper debug state was active at the time of VM-exit. */
280 bool fWasHyperDebugStateActive;
281 /** Whether TSC-offsetting should be setup before VM-entry. */
282 bool fUpdateTscOffsettingAndPreemptTimer;
283 /** Whether the VM-exit was caused by a page-fault during delivery of a
284 * contributory exception or a page-fault. */
285 bool fVectoringDoublePF;
286 /** Whether the VM-exit was caused by a page-fault during delivery of an
287 * external interrupt or NMI. */
288 bool fVectoringPF;
289} VMXTRANSIENT;
290AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
291AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
292AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
294AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
295/** Pointer to VMX transient state. */
296typedef VMXTRANSIENT *PVMXTRANSIENT;
297
298
299/**
300 * MSR-bitmap read permissions.
301 */
302typedef enum VMXMSREXITREAD
303{
304 /** Reading this MSR causes a VM-exit. */
305 VMXMSREXIT_INTERCEPT_READ = 0xb,
306 /** Reading this MSR does not cause a VM-exit. */
307 VMXMSREXIT_PASSTHRU_READ
308} VMXMSREXITREAD;
309/** Pointer to MSR-bitmap read permissions. */
310typedef VMXMSREXITREAD* PVMXMSREXITREAD;
311
312/**
313 * MSR-bitmap write permissions.
314 */
315typedef enum VMXMSREXITWRITE
316{
317 /** Writing to this MSR causes a VM-exit. */
318 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
319 /** Writing to this MSR does not cause a VM-exit. */
320 VMXMSREXIT_PASSTHRU_WRITE
321} VMXMSREXITWRITE;
322/** Pointer to MSR-bitmap write permissions. */
323typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
324
325
326/**
327 * VMX VM-exit handler.
328 *
329 * @returns VBox status code.
330 * @param pVCpu Pointer to the VMCPU.
331 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
332 * out-of-sync. Make sure to update the required
333 * fields before using them.
334 * @param pVmxTransient Pointer to the VMX-transient structure.
335 */
336#ifndef HMVMX_USE_FUNCTION_TABLE
337typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
338#else
339typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
340/** Pointer to VM-exit handler. */
341typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342#endif
343
344
345/*******************************************************************************
346* Internal Functions *
347*******************************************************************************/
348static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
349static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
350static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
351 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
352 bool fStepping, uint32_t *puIntState);
353#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
354static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
355#endif
356#ifndef HMVMX_USE_FUNCTION_TABLE
357DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
358# define HMVMX_EXIT_DECL static int
359#else
360# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
361#endif
362DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
363 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
364
365/** @name VM-exit handlers.
366 * @{
367 */
368static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
369static FNVMXEXITHANDLER hmR0VmxExitExtInt;
370static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
371static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
372static FNVMXEXITHANDLER hmR0VmxExitSipi;
373static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
374static FNVMXEXITHANDLER hmR0VmxExitSmi;
375static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
376static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
377static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
378static FNVMXEXITHANDLER hmR0VmxExitCpuid;
379static FNVMXEXITHANDLER hmR0VmxExitGetsec;
380static FNVMXEXITHANDLER hmR0VmxExitHlt;
381static FNVMXEXITHANDLER hmR0VmxExitInvd;
382static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
383static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
384static FNVMXEXITHANDLER hmR0VmxExitVmcall;
385static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
386static FNVMXEXITHANDLER hmR0VmxExitRsm;
387static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
388static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
389static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
390static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
391static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
392static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
393static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
394static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
395static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
396static FNVMXEXITHANDLER hmR0VmxExitMwait;
397static FNVMXEXITHANDLER hmR0VmxExitMtf;
398static FNVMXEXITHANDLER hmR0VmxExitMonitor;
399static FNVMXEXITHANDLER hmR0VmxExitPause;
400static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
401static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
402static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
403static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
404static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
405static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
406static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
407static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
408static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
409static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
410static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
411static FNVMXEXITHANDLER hmR0VmxExitRdrand;
412static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
413/** @} */
414
415static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
417static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
418static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
419static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
420static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
421#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
422static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
423#endif
424static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
425
426/*******************************************************************************
427* Global Variables *
428*******************************************************************************/
429#ifdef HMVMX_USE_FUNCTION_TABLE
430
431/**
432 * VMX_EXIT dispatch table.
433 */
434static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
435{
436 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
437 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
438 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
439 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
440 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
441 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
442 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
443 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
444 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
445 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
446 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
447 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
448 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
449 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
450 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
451 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
452 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
453 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
454 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
455 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
456 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
457 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
458 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
459 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
460 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
461 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
462 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
463 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
464 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
465 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
466 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
467 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
468 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
469 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
470 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
471 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
472 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
473 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
474 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
475 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
476 /* 40 UNDEFINED */ hmR0VmxExitPause,
477 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
478 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
479 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
480 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
481 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
482 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
483 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
484 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
485 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
486 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
487 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
488 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
489 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
490 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
491 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
492 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
493 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
494 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
495 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
496};
497#endif /* HMVMX_USE_FUNCTION_TABLE */
498
499#ifdef VBOX_STRICT
500static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
501{
502 /* 0 */ "(Not Used)",
503 /* 1 */ "VMCALL executed in VMX root operation.",
504 /* 2 */ "VMCLEAR with invalid physical address.",
505 /* 3 */ "VMCLEAR with VMXON pointer.",
506 /* 4 */ "VMLAUNCH with non-clear VMCS.",
507 /* 5 */ "VMRESUME with non-launched VMCS.",
508 /* 6 */ "VMRESUME after VMXOFF",
509 /* 7 */ "VM-entry with invalid control fields.",
510 /* 8 */ "VM-entry with invalid host state fields.",
511 /* 9 */ "VMPTRLD with invalid physical address.",
512 /* 10 */ "VMPTRLD with VMXON pointer.",
513 /* 11 */ "VMPTRLD with incorrect revision identifier.",
514 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
515 /* 13 */ "VMWRITE to read-only VMCS component.",
516 /* 14 */ "(Not Used)",
517 /* 15 */ "VMXON executed in VMX root operation.",
518 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
519 /* 17 */ "VM-entry with non-launched executing VMCS.",
520 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
521 /* 19 */ "VMCALL with non-clear VMCS.",
522 /* 20 */ "VMCALL with invalid VM-exit control fields.",
523 /* 21 */ "(Not Used)",
524 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
525 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
526 /* 24 */ "VMCALL with invalid SMM-monitor features.",
527 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
528 /* 26 */ "VM-entry with events blocked by MOV SS.",
529 /* 27 */ "(Not Used)",
530 /* 28 */ "Invalid operand to INVEPT/INVVPID."
531};
532#endif /* VBOX_STRICT */
533
534
535
536/**
537 * Updates the VM's last error record. If there was a VMX instruction error,
538 * reads the error data from the VMCS and updates VCPU's last error record as
539 * well.
540 *
541 * @param pVM Pointer to the VM.
542 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
543 * VERR_VMX_UNABLE_TO_START_VM or
544 * VERR_VMX_INVALID_VMCS_FIELD).
545 * @param rc The error code.
546 */
547static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
548{
549 AssertPtr(pVM);
550 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
551 || rc == VERR_VMX_UNABLE_TO_START_VM)
552 {
553 AssertPtrReturnVoid(pVCpu);
554 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
555 }
556 pVM->hm.s.lLastError = rc;
557}
558
559
560/**
561 * Reads the VM-entry interruption-information field from the VMCS into the VMX
562 * transient structure.
563 *
564 * @returns VBox status code.
565 * @param pVmxTransient Pointer to the VMX transient structure.
566 *
567 * @remarks No-long-jump zone!!!
568 */
569DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
570{
571 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
572 AssertRCReturn(rc, rc);
573 return VINF_SUCCESS;
574}
575
576
577/**
578 * Reads the VM-entry exception error code field from the VMCS into
579 * the VMX transient structure.
580 *
581 * @returns VBox status code.
582 * @param pVmxTransient Pointer to the VMX transient structure.
583 *
584 * @remarks No-long-jump zone!!!
585 */
586DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
587{
588 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
589 AssertRCReturn(rc, rc);
590 return VINF_SUCCESS;
591}
592
593
594/**
595 * Reads the VM-entry exception error code field from the VMCS into
596 * the VMX transient structure.
597 *
598 * @returns VBox status code.
599 * @param pVmxTransient Pointer to the VMX transient structure.
600 *
601 * @remarks No-long-jump zone!!!
602 */
603DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
604{
605 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
606 AssertRCReturn(rc, rc);
607 return VINF_SUCCESS;
608}
609
610
611/**
612 * Reads the VM-exit interruption-information field from the VMCS into the VMX
613 * transient structure.
614 *
615 * @returns VBox status code.
616 * @param pVmxTransient Pointer to the VMX transient structure.
617 */
618DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
619{
620 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
621 {
622 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
623 AssertRCReturn(rc, rc);
624 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
625 }
626 return VINF_SUCCESS;
627}
628
629
630/**
631 * Reads the VM-exit interruption error code from the VMCS into the VMX
632 * transient structure.
633 *
634 * @returns VBox status code.
635 * @param pVmxTransient Pointer to the VMX transient structure.
636 */
637DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
638{
639 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
640 {
641 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
642 AssertRCReturn(rc, rc);
643 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
644 }
645 return VINF_SUCCESS;
646}
647
648
649/**
650 * Reads the VM-exit instruction length field from the VMCS into the VMX
651 * transient structure.
652 *
653 * @returns VBox status code.
654 * @param pVCpu Pointer to the VMCPU.
655 * @param pVmxTransient Pointer to the VMX transient structure.
656 */
657DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
658{
659 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
660 {
661 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
662 AssertRCReturn(rc, rc);
663 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
664 }
665 return VINF_SUCCESS;
666}
667
668
669/**
670 * Reads the VM-exit instruction-information field from the VMCS into
671 * the VMX transient structure.
672 *
673 * @returns VBox status code.
674 * @param pVmxTransient Pointer to the VMX transient structure.
675 */
676DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
677{
678 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
679 {
680 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
681 AssertRCReturn(rc, rc);
682 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
683 }
684 return VINF_SUCCESS;
685}
686
687
688/**
689 * Reads the exit code qualification from the VMCS into the VMX transient
690 * structure.
691 *
692 * @returns VBox status code.
693 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
694 * case).
695 * @param pVmxTransient Pointer to the VMX transient structure.
696 */
697DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
698{
699 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
700 {
701 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
702 AssertRCReturn(rc, rc);
703 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
704 }
705 return VINF_SUCCESS;
706}
707
708
709/**
710 * Reads the IDT-vectoring information field from the VMCS into the VMX
711 * transient structure.
712 *
713 * @returns VBox status code.
714 * @param pVmxTransient Pointer to the VMX transient structure.
715 *
716 * @remarks No-long-jump zone!!!
717 */
718DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
719{
720 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
721 {
722 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
723 AssertRCReturn(rc, rc);
724 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
725 }
726 return VINF_SUCCESS;
727}
728
729
730/**
731 * Reads the IDT-vectoring error code from the VMCS into the VMX
732 * transient structure.
733 *
734 * @returns VBox status code.
735 * @param pVmxTransient Pointer to the VMX transient structure.
736 */
737DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
738{
739 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
740 {
741 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
742 AssertRCReturn(rc, rc);
743 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
744 }
745 return VINF_SUCCESS;
746}
747
748
749/**
750 * Enters VMX root mode operation on the current CPU.
751 *
752 * @returns VBox status code.
753 * @param pVM Pointer to the VM (optional, can be NULL, after
754 * a resume).
755 * @param HCPhysCpuPage Physical address of the VMXON region.
756 * @param pvCpuPage Pointer to the VMXON region.
757 */
758static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
759{
760 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
761 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
762 Assert(pvCpuPage);
763 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
764
765 if (pVM)
766 {
767 /* Write the VMCS revision dword to the VMXON region. */
768 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
769 }
770
771 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
772 RTCCUINTREG uEflags = ASMIntDisableFlags();
773
774 /* Enable the VMX bit in CR4 if necessary. */
775 RTCCUINTREG uCr4 = ASMGetCR4();
776 if (!(uCr4 & X86_CR4_VMXE))
777 ASMSetCR4(uCr4 | X86_CR4_VMXE);
778
779 /* Enter VMX root mode. */
780 int rc = VMXEnable(HCPhysCpuPage);
781 if (RT_FAILURE(rc))
782 ASMSetCR4(uCr4);
783
784 /* Restore interrupts. */
785 ASMSetFlags(uEflags);
786 return rc;
787}
788
789
790/**
791 * Exits VMX root mode operation on the current CPU.
792 *
793 * @returns VBox status code.
794 */
795static int hmR0VmxLeaveRootMode(void)
796{
797 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
798
799 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
800 RTCCUINTREG uEflags = ASMIntDisableFlags();
801
802 /* If we're for some reason not in VMX root mode, then don't leave it. */
803 RTCCUINTREG uHostCR4 = ASMGetCR4();
804
805 int rc;
806 if (uHostCR4 & X86_CR4_VMXE)
807 {
808 /* Exit VMX root mode and clear the VMX bit in CR4. */
809 VMXDisable();
810 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
811 rc = VINF_SUCCESS;
812 }
813 else
814 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
815
816 /* Restore interrupts. */
817 ASMSetFlags(uEflags);
818 return rc;
819}
820
821
822/**
823 * Allocates and maps one physically contiguous page. The allocated page is
824 * zero'd out. (Used by various VT-x structures).
825 *
826 * @returns IPRT status code.
827 * @param pMemObj Pointer to the ring-0 memory object.
828 * @param ppVirt Where to store the virtual address of the
829 * allocation.
830 * @param pPhys Where to store the physical address of the
831 * allocation.
832 */
833DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
834{
835 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
836 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
837 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
838
839 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
840 if (RT_FAILURE(rc))
841 return rc;
842 *ppVirt = RTR0MemObjAddress(*pMemObj);
843 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
844 ASMMemZero32(*ppVirt, PAGE_SIZE);
845 return VINF_SUCCESS;
846}
847
848
849/**
850 * Frees and unmaps an allocated physical page.
851 *
852 * @param pMemObj Pointer to the ring-0 memory object.
853 * @param ppVirt Where to re-initialize the virtual address of
854 * allocation as 0.
855 * @param pHCPhys Where to re-initialize the physical address of the
856 * allocation as 0.
857 */
858DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
859{
860 AssertPtr(pMemObj);
861 AssertPtr(ppVirt);
862 AssertPtr(pHCPhys);
863 if (*pMemObj != NIL_RTR0MEMOBJ)
864 {
865 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
866 AssertRC(rc);
867 *pMemObj = NIL_RTR0MEMOBJ;
868 *ppVirt = 0;
869 *pHCPhys = 0;
870 }
871}
872
873
874/**
875 * Worker function to free VT-x related structures.
876 *
877 * @returns IPRT status code.
878 * @param pVM Pointer to the VM.
879 */
880static void hmR0VmxStructsFree(PVM pVM)
881{
882 for (VMCPUID i = 0; i < pVM->cCpus; i++)
883 {
884 PVMCPU pVCpu = &pVM->aCpus[i];
885 AssertPtr(pVCpu);
886
887 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
888 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
889
890 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
891 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
892
893 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
894 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
895 }
896
897 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
898#ifdef VBOX_WITH_CRASHDUMP_MAGIC
899 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
900#endif
901}
902
903
904/**
905 * Worker function to allocate VT-x related VM structures.
906 *
907 * @returns IPRT status code.
908 * @param pVM Pointer to the VM.
909 */
910static int hmR0VmxStructsAlloc(PVM pVM)
911{
912 /*
913 * Initialize members up-front so we can cleanup properly on allocation failure.
914 */
915#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
916 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
917 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
918 pVM->hm.s.vmx.HCPhys##a_Name = 0;
919
920#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
921 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
922 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
923 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
924
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
927#endif
928 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
929
930 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
931 for (VMCPUID i = 0; i < pVM->cCpus; i++)
932 {
933 PVMCPU pVCpu = &pVM->aCpus[i];
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
935 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
936 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
937 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
938 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
939 }
940#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
941#undef VMXLOCAL_INIT_VM_MEMOBJ
942
943 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
944 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
945 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
946 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
947
948 /*
949 * Allocate all the VT-x structures.
950 */
951 int rc = VINF_SUCCESS;
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
957 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
958#endif
959
960 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
961 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
962 {
963 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
964 &pVM->hm.s.vmx.HCPhysApicAccess);
965 if (RT_FAILURE(rc))
966 goto cleanup;
967 }
968
969 /*
970 * Initialize per-VCPU VT-x structures.
971 */
972 for (VMCPUID i = 0; i < pVM->cCpus; i++)
973 {
974 PVMCPU pVCpu = &pVM->aCpus[i];
975 AssertPtr(pVCpu);
976
977 /* Allocate the VM control structure (VMCS). */
978 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
979 if (RT_FAILURE(rc))
980 goto cleanup;
981
982 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
983 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
984 {
985 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
986 &pVCpu->hm.s.vmx.HCPhysVirtApic);
987 if (RT_FAILURE(rc))
988 goto cleanup;
989 }
990
991 /*
992 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
993 * transparent accesses of specific MSRs.
994 *
995 * If the condition for enabling MSR bitmaps changes here, don't forget to
996 * update HMIsMsrBitmapsAvailable().
997 */
998 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
999 {
1000 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1001 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1002 if (RT_FAILURE(rc))
1003 goto cleanup;
1004 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1005 }
1006
1007 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1008 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1009 if (RT_FAILURE(rc))
1010 goto cleanup;
1011
1012 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1013 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016 }
1017
1018 return VINF_SUCCESS;
1019
1020cleanup:
1021 hmR0VmxStructsFree(pVM);
1022 return rc;
1023}
1024
1025
1026/**
1027 * Does global VT-x initialization (called during module initialization).
1028 *
1029 * @returns VBox status code.
1030 */
1031VMMR0DECL(int) VMXR0GlobalInit(void)
1032{
1033#ifdef HMVMX_USE_FUNCTION_TABLE
1034 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1035# ifdef VBOX_STRICT
1036 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1037 Assert(g_apfnVMExitHandlers[i]);
1038# endif
1039#endif
1040 return VINF_SUCCESS;
1041}
1042
1043
1044/**
1045 * Does global VT-x termination (called during module termination).
1046 */
1047VMMR0DECL(void) VMXR0GlobalTerm()
1048{
1049 /* Nothing to do currently. */
1050}
1051
1052
1053/**
1054 * Sets up and activates VT-x on the current CPU.
1055 *
1056 * @returns VBox status code.
1057 * @param pCpu Pointer to the global CPU info struct.
1058 * @param pVM Pointer to the VM (can be NULL after a host resume
1059 * operation).
1060 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1061 * fEnabledByHost is true).
1062 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1063 * @a fEnabledByHost is true).
1064 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1065 * enable VT-x on the host.
1066 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1067 */
1068VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1069 void *pvMsrs)
1070{
1071 Assert(pCpu);
1072 Assert(pvMsrs);
1073 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1074
1075 /* Enable VT-x if it's not already enabled by the host. */
1076 if (!fEnabledByHost)
1077 {
1078 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1079 if (RT_FAILURE(rc))
1080 return rc;
1081 }
1082
1083 /*
1084 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1085 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1086 */
1087 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1088 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1089 {
1090 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1091 pCpu->fFlushAsidBeforeUse = false;
1092 }
1093 else
1094 pCpu->fFlushAsidBeforeUse = true;
1095
1096 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1097 ++pCpu->cTlbFlushes;
1098
1099 return VINF_SUCCESS;
1100}
1101
1102
1103/**
1104 * Deactivates VT-x on the current CPU.
1105 *
1106 * @returns VBox status code.
1107 * @param pCpu Pointer to the global CPU info struct.
1108 * @param pvCpuPage Pointer to the VMXON region.
1109 * @param HCPhysCpuPage Physical address of the VMXON region.
1110 *
1111 * @remarks This function should never be called when SUPR0EnableVTx() or
1112 * similar was used to enable VT-x on the host.
1113 */
1114VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1115{
1116 NOREF(pCpu);
1117 NOREF(pvCpuPage);
1118 NOREF(HCPhysCpuPage);
1119
1120 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1121 return hmR0VmxLeaveRootMode();
1122}
1123
1124
1125/**
1126 * Sets the permission bits for the specified MSR in the MSR bitmap.
1127 *
1128 * @param pVCpu Pointer to the VMCPU.
1129 * @param uMSR The MSR value.
1130 * @param enmRead Whether reading this MSR causes a VM-exit.
1131 * @param enmWrite Whether writing this MSR causes a VM-exit.
1132 */
1133static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1134{
1135 int32_t iBit;
1136 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1137
1138 /*
1139 * Layout:
1140 * 0x000 - 0x3ff - Low MSR read bits
1141 * 0x400 - 0x7ff - High MSR read bits
1142 * 0x800 - 0xbff - Low MSR write bits
1143 * 0xc00 - 0xfff - High MSR write bits
1144 */
1145 if (uMsr <= 0x00001FFF)
1146 iBit = uMsr;
1147 else if ( uMsr >= 0xC0000000
1148 && uMsr <= 0xC0001FFF)
1149 {
1150 iBit = (uMsr - 0xC0000000);
1151 pbMsrBitmap += 0x400;
1152 }
1153 else
1154 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1155
1156 Assert(iBit <= 0x1fff);
1157 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1158 ASMBitSet(pbMsrBitmap, iBit);
1159 else
1160 ASMBitClear(pbMsrBitmap, iBit);
1161
1162 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1163 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1164 else
1165 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1166}
1167
1168
1169#ifdef VBOX_STRICT
1170/**
1171 * Gets the permission bits for the specified MSR in the MSR bitmap.
1172 *
1173 * @returns VBox status code.
1174 * @retval VINF_SUCCESS if the specified MSR is found.
1175 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1176 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1177 *
1178 * @param pVCpu Pointer to the VMCPU.
1179 * @param uMsr The MSR.
1180 * @param penmRead Where to store the read permissions.
1181 * @param penmWrite Where to store the write permissions.
1182 */
1183static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1184{
1185 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1186 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1187 int32_t iBit;
1188 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1189
1190 /* See hmR0VmxSetMsrPermission() for the layout. */
1191 if (uMsr <= 0x00001FFF)
1192 iBit = uMsr;
1193 else if ( uMsr >= 0xC0000000
1194 && uMsr <= 0xC0001FFF)
1195 {
1196 iBit = (uMsr - 0xC0000000);
1197 pbMsrBitmap += 0x400;
1198 }
1199 else
1200 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1201
1202 Assert(iBit <= 0x1fff);
1203 if (ASMBitTest(pbMsrBitmap, iBit))
1204 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1205 else
1206 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1207
1208 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1209 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1210 else
1211 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1212 return VINF_SUCCESS;
1213}
1214#endif /* VBOX_STRICT */
1215
1216
1217/**
1218 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1219 * area.
1220 *
1221 * @returns VBox status code.
1222 * @param pVCpu Pointer to the VMCPU.
1223 * @param cMsrs The number of MSRs.
1224 */
1225DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1226{
1227 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1228 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1229 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1230 {
1231 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1232 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1233 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1234 }
1235
1236 /* Update number of guest MSRs to load/store across the world-switch. */
1237 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1238 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1239
1240 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1241 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1242
1243 /* Update the VCPU's copy of the MSR count. */
1244 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1245
1246 return VINF_SUCCESS;
1247}
1248
1249
1250/**
1251 * Adds a new (or updates the value of an existing) guest/host MSR
1252 * pair to be swapped during the world-switch as part of the
1253 * auto-load/store MSR area in the VMCS.
1254 *
1255 * @returns true if the MSR was added -and- its value was updated, false
1256 * otherwise.
1257 * @param pVCpu Pointer to the VMCPU.
1258 * @param uMsr The MSR.
1259 * @param uGuestMsr Value of the guest MSR.
1260 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1261 * necessary.
1262 */
1263static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1264{
1265 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1266 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1267 uint32_t i;
1268 for (i = 0; i < cMsrs; i++)
1269 {
1270 if (pGuestMsr->u32Msr == uMsr)
1271 break;
1272 pGuestMsr++;
1273 }
1274
1275 bool fAdded = false;
1276 if (i == cMsrs)
1277 {
1278 ++cMsrs;
1279 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1280 AssertRC(rc);
1281
1282 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1283 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1284 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1285
1286 fAdded = true;
1287 }
1288
1289 /* Update the MSR values in the auto-load/store MSR area. */
1290 pGuestMsr->u32Msr = uMsr;
1291 pGuestMsr->u64Value = uGuestMsrValue;
1292
1293 /* Create/update the MSR slot in the host MSR area. */
1294 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1295 pHostMsr += i;
1296 pHostMsr->u32Msr = uMsr;
1297
1298 /*
1299 * Update the host MSR only when requested by the caller AND when we're
1300 * adding it to the auto-load/store area. Otherwise, it would have been
1301 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1302 */
1303 bool fUpdatedMsrValue = false;
1304 if ( fAdded
1305 && fUpdateHostMsr)
1306 {
1307 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1308 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1309 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1310 fUpdatedMsrValue = true;
1311 }
1312
1313 return fUpdatedMsrValue;
1314}
1315
1316
1317/**
1318 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1319 * auto-load/store MSR area in the VMCS.
1320 *
1321 * @returns VBox status code.
1322 * @param pVCpu Pointer to the VMCPU.
1323 * @param uMsr The MSR.
1324 */
1325static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1326{
1327 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1328 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1329 for (uint32_t i = 0; i < cMsrs; i++)
1330 {
1331 /* Find the MSR. */
1332 if (pGuestMsr->u32Msr == uMsr)
1333 {
1334 /* If it's the last MSR, simply reduce the count. */
1335 if (i == cMsrs - 1)
1336 {
1337 --cMsrs;
1338 break;
1339 }
1340
1341 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1342 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1343 pLastGuestMsr += cMsrs - 1;
1344 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1345 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1346
1347 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1348 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1349 pLastHostMsr += cMsrs - 1;
1350 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1351 pHostMsr->u64Value = pLastHostMsr->u64Value;
1352 --cMsrs;
1353 break;
1354 }
1355 pGuestMsr++;
1356 }
1357
1358 /* Update the VMCS if the count changed (meaning the MSR was found). */
1359 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1360 {
1361 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1362 AssertRCReturn(rc, rc);
1363
1364 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1365 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1366 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1367
1368 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1369 return VINF_SUCCESS;
1370 }
1371
1372 return VERR_NOT_FOUND;
1373}
1374
1375
1376/**
1377 * Checks if the specified guest MSR is part of the auto-load/store area in
1378 * the VMCS.
1379 *
1380 * @returns true if found, false otherwise.
1381 * @param pVCpu Pointer to the VMCPU.
1382 * @param uMsr The MSR to find.
1383 */
1384static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1385{
1386 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1387 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1388
1389 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1390 {
1391 if (pGuestMsr->u32Msr == uMsr)
1392 return true;
1393 }
1394 return false;
1395}
1396
1397
1398/**
1399 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1400 *
1401 * @param pVCpu Pointer to the VMCPU.
1402 *
1403 * @remarks No-long-jump zone!!!
1404 */
1405static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1406{
1407 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1408 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1409 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1410 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1411
1412 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1413 {
1414 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1415
1416 /*
1417 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1418 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1419 */
1420 if (pHostMsr->u32Msr == MSR_K6_EFER)
1421 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1422 else
1423 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1424 }
1425
1426 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1427}
1428
1429
1430#if HC_ARCH_BITS == 64
1431/**
1432 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1433 * perform lazy restoration of the host MSRs while leaving VT-x.
1434 *
1435 * @param pVCpu Pointer to the VMCPU.
1436 *
1437 * @remarks No-long-jump zone!!!
1438 */
1439static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1440{
1441 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1442
1443 /*
1444 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1445 */
1446 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1447 {
1448 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1449 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1450 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1451 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1452 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1453 }
1454}
1455
1456
1457/**
1458 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1459 * lazily while leaving VT-x.
1460 *
1461 * @returns true if it does, false otherwise.
1462 * @param pVCpu Pointer to the VMCPU.
1463 * @param uMsr The MSR to check.
1464 */
1465static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1466{
1467 NOREF(pVCpu);
1468 switch (uMsr)
1469 {
1470 case MSR_K8_LSTAR:
1471 case MSR_K6_STAR:
1472 case MSR_K8_SF_MASK:
1473 case MSR_K8_KERNEL_GS_BASE:
1474 return true;
1475 }
1476 return false;
1477}
1478
1479
1480/**
1481 * Saves a set of guest MSRs back into the guest-CPU context.
1482 *
1483 * @param pVCpu Pointer to the VMCPU.
1484 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1485 * out-of-sync. Make sure to update the required fields
1486 * before using them.
1487 *
1488 * @remarks No-long-jump zone!!!
1489 */
1490static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1491{
1492 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1493 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1494
1495 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1496 {
1497 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1498 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1499 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1500 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1501 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1502 }
1503}
1504
1505
1506/**
1507 * Loads a set of guests MSRs to allow read/passthru to the guest.
1508 *
1509 * The name of this function is slightly confusing. This function does NOT
1510 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1511 * common prefix for functions dealing with "lazy restoration" of the shared
1512 * MSRs.
1513 *
1514 * @param pVCpu Pointer to the VMCPU.
1515 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1516 * out-of-sync. Make sure to update the required fields
1517 * before using them.
1518 *
1519 * @remarks No-long-jump zone!!!
1520 */
1521static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1522{
1523 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1524 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1525
1526#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1527 do { \
1528 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1529 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1530 else \
1531 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1532 } while (0)
1533
1534 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1535 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1536 {
1537 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1538 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1539 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1540 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1541 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1542 }
1543 else
1544 {
1545 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1546 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1547 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1548 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1549 }
1550
1551#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1552}
1553
1554
1555/**
1556 * Performs lazy restoration of the set of host MSRs if they were previously
1557 * loaded with guest MSR values.
1558 *
1559 * @param pVCpu Pointer to the VMCPU.
1560 *
1561 * @remarks No-long-jump zone!!!
1562 * @remarks The guest MSRs should have been saved back into the guest-CPU
1563 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1564 */
1565static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1566{
1567 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1568 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1569
1570 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1571 {
1572 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1573 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1574 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1575 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1576 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1577 }
1578 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1579}
1580#endif /* HC_ARCH_BITS == 64 */
1581
1582
1583/**
1584 * Verifies that our cached values of the VMCS controls are all
1585 * consistent with what's actually present in the VMCS.
1586 *
1587 * @returns VBox status code.
1588 * @param pVCpu Pointer to the VMCPU.
1589 */
1590static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1591{
1592 uint32_t u32Val;
1593 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1594 AssertRCReturn(rc, rc);
1595 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1596 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1597
1598 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1599 AssertRCReturn(rc, rc);
1600 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1601 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1602
1603 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1604 AssertRCReturn(rc, rc);
1605 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1606 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1607
1608 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1609 AssertRCReturn(rc, rc);
1610 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1611 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1612
1613 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1614 {
1615 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1616 AssertRCReturn(rc, rc);
1617 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1618 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1619 }
1620
1621 return VINF_SUCCESS;
1622}
1623
1624
1625#ifdef VBOX_STRICT
1626/**
1627 * Verifies that our cached host EFER value has not changed
1628 * since we cached it.
1629 *
1630 * @param pVCpu Pointer to the VMCPU.
1631 */
1632static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1633{
1634 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1635
1636 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1637 {
1638 uint64_t u64Val;
1639 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1640 AssertRC(rc);
1641
1642 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1643 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1644 }
1645}
1646
1647
1648/**
1649 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1650 * VMCS are correct.
1651 *
1652 * @param pVCpu Pointer to the VMCPU.
1653 */
1654static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1655{
1656 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1657
1658 /* Verify MSR counts in the VMCS are what we think it should be. */
1659 uint32_t cMsrs;
1660 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1661 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1662
1663 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1664 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1665
1666 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1667 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1668
1669 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1670 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1671 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1672 {
1673 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1674 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1675 pGuestMsr->u32Msr, cMsrs));
1676
1677 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1678 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1679 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1680
1681 /* Verify that the permissions are as expected in the MSR bitmap. */
1682 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1683 {
1684 VMXMSREXITREAD enmRead;
1685 VMXMSREXITWRITE enmWrite;
1686 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1687 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1688 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1689 {
1690 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1691 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1692 }
1693 else
1694 {
1695 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1696 pGuestMsr->u32Msr, cMsrs));
1697 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1698 pGuestMsr->u32Msr, cMsrs));
1699 }
1700 }
1701 }
1702}
1703#endif /* VBOX_STRICT */
1704
1705
1706/**
1707 * Flushes the TLB using EPT.
1708 *
1709 * @returns VBox status code.
1710 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1711 * enmFlush).
1712 * @param enmFlush Type of flush.
1713 *
1714 * @remarks Caller is responsible for making sure this function is called only
1715 * when NestedPaging is supported and providing @a enmFlush that is
1716 * supported by the CPU.
1717 * @remarks Can be called with interrupts disabled.
1718 */
1719static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1720{
1721 uint64_t au64Descriptor[2];
1722 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1723 au64Descriptor[0] = 0;
1724 else
1725 {
1726 Assert(pVCpu);
1727 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1728 }
1729 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1730
1731 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1732 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1733 rc));
1734 if ( RT_SUCCESS(rc)
1735 && pVCpu)
1736 {
1737 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1738 }
1739}
1740
1741
1742/**
1743 * Flushes the TLB using VPID.
1744 *
1745 * @returns VBox status code.
1746 * @param pVM Pointer to the VM.
1747 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1748 * enmFlush).
1749 * @param enmFlush Type of flush.
1750 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1751 * on @a enmFlush).
1752 *
1753 * @remarks Can be called with interrupts disabled.
1754 */
1755static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1756{
1757 NOREF(pVM);
1758 AssertPtr(pVM);
1759 Assert(pVM->hm.s.vmx.fVpid);
1760
1761 uint64_t au64Descriptor[2];
1762 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1763 {
1764 au64Descriptor[0] = 0;
1765 au64Descriptor[1] = 0;
1766 }
1767 else
1768 {
1769 AssertPtr(pVCpu);
1770 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1771 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1772 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1773 au64Descriptor[1] = GCPtr;
1774 }
1775
1776 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1777 AssertMsg(rc == VINF_SUCCESS,
1778 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1779 if ( RT_SUCCESS(rc)
1780 && pVCpu)
1781 {
1782 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1783 }
1784}
1785
1786
1787/**
1788 * Invalidates a guest page by guest virtual address. Only relevant for
1789 * EPT/VPID, otherwise there is nothing really to invalidate.
1790 *
1791 * @returns VBox status code.
1792 * @param pVM Pointer to the VM.
1793 * @param pVCpu Pointer to the VMCPU.
1794 * @param GCVirt Guest virtual address of the page to invalidate.
1795 */
1796VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1797{
1798 AssertPtr(pVM);
1799 AssertPtr(pVCpu);
1800 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1801
1802 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1803 if (!fFlushPending)
1804 {
1805 /*
1806 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1807 * See @bugref{6043} and @bugref{6177}.
1808 *
1809 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1810 * function maybe called in a loop with individual addresses.
1811 */
1812 if (pVM->hm.s.vmx.fVpid)
1813 {
1814 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1815 {
1816 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1817 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1818 }
1819 else
1820 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1821 }
1822 else if (pVM->hm.s.fNestedPaging)
1823 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1824 }
1825
1826 return VINF_SUCCESS;
1827}
1828
1829
1830/**
1831 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1832 * otherwise there is nothing really to invalidate.
1833 *
1834 * @returns VBox status code.
1835 * @param pVM Pointer to the VM.
1836 * @param pVCpu Pointer to the VMCPU.
1837 * @param GCPhys Guest physical address of the page to invalidate.
1838 */
1839VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1840{
1841 NOREF(pVM); NOREF(GCPhys);
1842 LogFlowFunc(("%RGp\n", GCPhys));
1843
1844 /*
1845 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1846 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1847 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1848 */
1849 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1850 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1851 return VINF_SUCCESS;
1852}
1853
1854
1855/**
1856 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1857 * case where neither EPT nor VPID is supported by the CPU.
1858 *
1859 * @param pVM Pointer to the VM.
1860 * @param pVCpu Pointer to the VMCPU.
1861 * @param pCpu Pointer to the global HM struct.
1862 *
1863 * @remarks Called with interrupts disabled.
1864 */
1865static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1866{
1867 AssertPtr(pVCpu);
1868 AssertPtr(pCpu);
1869 NOREF(pVM);
1870
1871 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1872
1873 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1874#if 0
1875 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1876 pVCpu->hm.s.TlbShootdown.cPages = 0;
1877#endif
1878
1879 Assert(pCpu->idCpu != NIL_RTCPUID);
1880 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1881 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1882 pVCpu->hm.s.fForceTLBFlush = false;
1883 return;
1884}
1885
1886
1887/**
1888 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1889 *
1890 * @param pVM Pointer to the VM.
1891 * @param pVCpu Pointer to the VMCPU.
1892 * @param pCpu Pointer to the global HM CPU struct.
1893 * @remarks All references to "ASID" in this function pertains to "VPID" in
1894 * Intel's nomenclature. The reason is, to avoid confusion in compare
1895 * statements since the host-CPU copies are named "ASID".
1896 *
1897 * @remarks Called with interrupts disabled.
1898 */
1899static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1900{
1901#ifdef VBOX_WITH_STATISTICS
1902 bool fTlbFlushed = false;
1903# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1904# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1905 if (!fTlbFlushed) \
1906 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1907 } while (0)
1908#else
1909# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1910# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1911#endif
1912
1913 AssertPtr(pVM);
1914 AssertPtr(pCpu);
1915 AssertPtr(pVCpu);
1916 Assert(pCpu->idCpu != NIL_RTCPUID);
1917
1918 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1919 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1920 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1921
1922 /*
1923 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1924 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1925 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1926 */
1927 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1928 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1929 {
1930 ++pCpu->uCurrentAsid;
1931 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1932 {
1933 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1934 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1935 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1936 }
1937
1938 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1939 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1940 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1941
1942 /*
1943 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1944 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1945 */
1946 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1947 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1948 HMVMX_SET_TAGGED_TLB_FLUSHED();
1949 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1950 }
1951
1952 /* Check for explicit TLB shootdowns. */
1953 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1954 {
1955 /*
1956 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1957 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1958 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1959 * but not guest-physical mappings.
1960 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1961 */
1962 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1963 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1964 HMVMX_SET_TAGGED_TLB_FLUSHED();
1965 }
1966
1967 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1968 * where it is commented out. Support individual entry flushing
1969 * someday. */
1970#if 0
1971 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1972 {
1973 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1974
1975 /*
1976 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1977 * as supported by the CPU.
1978 */
1979 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1980 {
1981 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1982 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1983 }
1984 else
1985 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1986
1987 HMVMX_SET_TAGGED_TLB_FLUSHED();
1988 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1989 pVCpu->hm.s.TlbShootdown.cPages = 0;
1990 }
1991#endif
1992
1993 pVCpu->hm.s.fForceTLBFlush = false;
1994
1995 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1996
1997 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1998 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1999 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2000 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2001 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2002 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2003 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2004 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2005 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2006
2007 /* Update VMCS with the VPID. */
2008 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2009 AssertRC(rc);
2010
2011#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2012}
2013
2014
2015/**
2016 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2017 *
2018 * @returns VBox status code.
2019 * @param pVM Pointer to the VM.
2020 * @param pVCpu Pointer to the VMCPU.
2021 * @param pCpu Pointer to the global HM CPU struct.
2022 *
2023 * @remarks Called with interrupts disabled.
2024 */
2025static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2026{
2027 AssertPtr(pVM);
2028 AssertPtr(pVCpu);
2029 AssertPtr(pCpu);
2030 Assert(pCpu->idCpu != NIL_RTCPUID);
2031 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2032 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2033
2034 /*
2035 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2036 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2037 */
2038 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2039 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2040 {
2041 pVCpu->hm.s.fForceTLBFlush = true;
2042 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2043 }
2044
2045 /* Check for explicit TLB shootdown flushes. */
2046 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2047 {
2048 pVCpu->hm.s.fForceTLBFlush = true;
2049 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2050 }
2051
2052 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2053 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2054
2055 if (pVCpu->hm.s.fForceTLBFlush)
2056 {
2057 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2058 pVCpu->hm.s.fForceTLBFlush = false;
2059 }
2060 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2061 * where it is commented out. Support individual entry flushing
2062 * someday. */
2063#if 0
2064 else
2065 {
2066 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2067 {
2068 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2069 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2070 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2071 }
2072 else
2073 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2074
2075 pVCpu->hm.s.TlbShootdown.cPages = 0;
2076 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2077 }
2078#endif
2079}
2080
2081
2082/**
2083 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2084 *
2085 * @returns VBox status code.
2086 * @param pVM Pointer to the VM.
2087 * @param pVCpu Pointer to the VMCPU.
2088 * @param pCpu Pointer to the global HM CPU struct.
2089 *
2090 * @remarks Called with interrupts disabled.
2091 */
2092static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2093{
2094 AssertPtr(pVM);
2095 AssertPtr(pVCpu);
2096 AssertPtr(pCpu);
2097 Assert(pCpu->idCpu != NIL_RTCPUID);
2098 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2099 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2100
2101 /*
2102 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2103 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2104 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2105 */
2106 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2107 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2108 {
2109 pVCpu->hm.s.fForceTLBFlush = true;
2110 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2111 }
2112
2113 /* Check for explicit TLB shootdown flushes. */
2114 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2115 {
2116 /*
2117 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2118 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2119 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2120 */
2121 pVCpu->hm.s.fForceTLBFlush = true;
2122 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2123 }
2124
2125 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2126 if (pVCpu->hm.s.fForceTLBFlush)
2127 {
2128 ++pCpu->uCurrentAsid;
2129 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2130 {
2131 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2132 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2133 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2134 }
2135
2136 pVCpu->hm.s.fForceTLBFlush = false;
2137 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2138 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2139 if (pCpu->fFlushAsidBeforeUse)
2140 {
2141 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2142 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2143 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2144 {
2145 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2146 pCpu->fFlushAsidBeforeUse = false;
2147 }
2148 else
2149 {
2150 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2151 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2152 }
2153 }
2154 }
2155 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2156 * where it is commented out. Support individual entry flushing
2157 * someday. */
2158#if 0
2159 else
2160 {
2161 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2162 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2163 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2164 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2165
2166 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2167 {
2168 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2169 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2170 {
2171 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2172 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2173 }
2174 else
2175 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2176
2177 pVCpu->hm.s.TlbShootdown.cPages = 0;
2178 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2179 }
2180 else
2181 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2182 }
2183#endif
2184
2185 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2186 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2187 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2188 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2189 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2190 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2191 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2192
2193 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2194 AssertRC(rc);
2195}
2196
2197
2198/**
2199 * Flushes the guest TLB entry based on CPU capabilities.
2200 *
2201 * @param pVCpu Pointer to the VMCPU.
2202 * @param pCpu Pointer to the global HM CPU struct.
2203 */
2204DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2205{
2206#ifdef HMVMX_ALWAYS_FLUSH_TLB
2207 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2208#endif
2209 PVM pVM = pVCpu->CTX_SUFF(pVM);
2210 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2211 {
2212 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2213 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2214 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2215 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2216 default:
2217 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2218 break;
2219 }
2220
2221 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2222 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2223
2224 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2225}
2226
2227
2228/**
2229 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2230 * TLB entries from the host TLB before VM-entry.
2231 *
2232 * @returns VBox status code.
2233 * @param pVM Pointer to the VM.
2234 */
2235static int hmR0VmxSetupTaggedTlb(PVM pVM)
2236{
2237 /*
2238 * Determine optimal flush type for Nested Paging.
2239 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2240 * guest execution (see hmR3InitFinalizeR0()).
2241 */
2242 if (pVM->hm.s.fNestedPaging)
2243 {
2244 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2245 {
2246 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2247 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2248 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2249 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2250 else
2251 {
2252 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2253 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2254 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2255 }
2256
2257 /* Make sure the write-back cacheable memory type for EPT is supported. */
2258 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2259 {
2260 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2261 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2262 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2263 }
2264 }
2265 else
2266 {
2267 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2268 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2269 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2270 }
2271 }
2272
2273 /*
2274 * Determine optimal flush type for VPID.
2275 */
2276 if (pVM->hm.s.vmx.fVpid)
2277 {
2278 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2279 {
2280 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2281 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2282 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2283 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2284 else
2285 {
2286 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2287 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2288 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2289 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2290 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2291 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2292 pVM->hm.s.vmx.fVpid = false;
2293 }
2294 }
2295 else
2296 {
2297 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2298 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2299 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2300 pVM->hm.s.vmx.fVpid = false;
2301 }
2302 }
2303
2304 /*
2305 * Setup the handler for flushing tagged-TLBs.
2306 */
2307 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2308 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2309 else if (pVM->hm.s.fNestedPaging)
2310 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2311 else if (pVM->hm.s.vmx.fVpid)
2312 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2313 else
2314 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2315 return VINF_SUCCESS;
2316}
2317
2318
2319/**
2320 * Sets up pin-based VM-execution controls in the VMCS.
2321 *
2322 * @returns VBox status code.
2323 * @param pVM Pointer to the VM.
2324 * @param pVCpu Pointer to the VMCPU.
2325 */
2326static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2327{
2328 AssertPtr(pVM);
2329 AssertPtr(pVCpu);
2330
2331 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2332 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2333
2334 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2335 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2336
2337 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2338 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2339
2340 /* Enable the VMX preemption timer. */
2341 if (pVM->hm.s.vmx.fUsePreemptTimer)
2342 {
2343 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2344 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2345 }
2346
2347 if ((val & zap) != val)
2348 {
2349 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2350 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2351 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2352 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2353 }
2354
2355 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2356 AssertRCReturn(rc, rc);
2357
2358 pVCpu->hm.s.vmx.u32PinCtls = val;
2359 return rc;
2360}
2361
2362
2363/**
2364 * Sets up processor-based VM-execution controls in the VMCS.
2365 *
2366 * @returns VBox status code.
2367 * @param pVM Pointer to the VM.
2368 * @param pVMCPU Pointer to the VMCPU.
2369 */
2370static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2371{
2372 AssertPtr(pVM);
2373 AssertPtr(pVCpu);
2374
2375 int rc = VERR_INTERNAL_ERROR_5;
2376 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2377 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2378
2379 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2380 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2381 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2382 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2383 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2384 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2385 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2386
2387 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2388 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2389 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2390 {
2391 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2392 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2393 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2394 }
2395
2396 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2397 if (!pVM->hm.s.fNestedPaging)
2398 {
2399 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2400 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2401 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2402 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2403 }
2404
2405 /* Use TPR shadowing if supported by the CPU. */
2406 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2407 {
2408 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2409 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2410 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2411 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2412 AssertRCReturn(rc, rc);
2413
2414 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2415 /* CR8 writes cause a VM-exit based on TPR threshold. */
2416 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2417 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2418 }
2419 else
2420 {
2421 /*
2422 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2423 * Set this control only for 64-bit guests.
2424 */
2425 if (pVM->hm.s.fAllow64BitGuests)
2426 {
2427 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2428 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2429 }
2430 }
2431
2432 /* Use MSR-bitmaps if supported by the CPU. */
2433 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2434 {
2435 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2436
2437 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2438 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2439 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2440 AssertRCReturn(rc, rc);
2441
2442 /*
2443 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2444 * automatically using dedicated fields in the VMCS.
2445 */
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2449 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451
2452#if HC_ARCH_BITS == 64
2453 /*
2454 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2455 */
2456 if (pVM->hm.s.fAllow64BitGuests)
2457 {
2458 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2459 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2460 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2461 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2462 }
2463#endif
2464 }
2465
2466 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2467 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2468 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2469
2470 if ((val & zap) != val)
2471 {
2472 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2473 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2474 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2475 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2476 }
2477
2478 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2479 AssertRCReturn(rc, rc);
2480
2481 pVCpu->hm.s.vmx.u32ProcCtls = val;
2482
2483 /*
2484 * Secondary processor-based VM-execution controls.
2485 */
2486 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2487 {
2488 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2489 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2490
2491 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2492 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2493
2494 if (pVM->hm.s.fNestedPaging)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2496 else
2497 {
2498 /*
2499 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2500 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2501 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2502 */
2503 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2504 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2505 }
2506
2507 if (pVM->hm.s.vmx.fVpid)
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2509
2510 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2511 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2512
2513 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515 * done dynamically. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517 {
2518 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522 AssertRCReturn(rc, rc);
2523 }
2524
2525 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528 if ((val & zap) != val)
2529 {
2530 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2531 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2532 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2533 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2534 }
2535
2536 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2537 AssertRCReturn(rc, rc);
2538
2539 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2540 }
2541 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2542 {
2543 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2544 "available\n"));
2545 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2546 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2547 }
2548
2549 return VINF_SUCCESS;
2550}
2551
2552
2553/**
2554 * Sets up miscellaneous (everything other than Pin & Processor-based
2555 * VM-execution) control fields in the VMCS.
2556 *
2557 * @returns VBox status code.
2558 * @param pVM Pointer to the VM.
2559 * @param pVCpu Pointer to the VMCPU.
2560 */
2561static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2562{
2563 NOREF(pVM);
2564 AssertPtr(pVM);
2565 AssertPtr(pVCpu);
2566
2567 int rc = VERR_GENERAL_FAILURE;
2568
2569 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2570#if 0
2571 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2572 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2573 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2574
2575 /*
2576 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2577 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2578 * We thus use the exception bitmap to control it rather than use both.
2579 */
2580 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2581 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2582
2583 /** @todo Explore possibility of using IO-bitmaps. */
2584 /* All IO & IOIO instructions cause VM-exits. */
2585 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2586 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2587
2588 /* Initialize the MSR-bitmap area. */
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2590 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2592#endif
2593
2594 /* Setup MSR auto-load/store area. */
2595 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2596 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2597 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2598 AssertRCReturn(rc, rc);
2599 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2600 AssertRCReturn(rc, rc);
2601
2602 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2603 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2604 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2605 AssertRCReturn(rc, rc);
2606
2607 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2608 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2609 AssertRCReturn(rc, rc);
2610
2611 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2612#if 0
2613 /* Setup debug controls */
2614 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2615 AssertRCReturn(rc, rc);
2616 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2617 AssertRCReturn(rc, rc);
2618#endif
2619
2620 return rc;
2621}
2622
2623
2624/**
2625 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2626 *
2627 * @returns VBox status code.
2628 * @param pVM Pointer to the VM.
2629 * @param pVCpu Pointer to the VMCPU.
2630 */
2631static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2632{
2633 AssertPtr(pVM);
2634 AssertPtr(pVCpu);
2635
2636 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2637
2638 uint32_t u32XcptBitmap = 0;
2639
2640 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2641 if (!pVM->hm.s.fNestedPaging)
2642 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2643
2644 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2645 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2646 AssertRCReturn(rc, rc);
2647 return rc;
2648}
2649
2650
2651/**
2652 * Sets up the initial guest-state mask. The guest-state mask is consulted
2653 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2654 * for the nested virtualization case (as it would cause a VM-exit).
2655 *
2656 * @param pVCpu Pointer to the VMCPU.
2657 */
2658static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2659{
2660 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2661 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2662 return VINF_SUCCESS;
2663}
2664
2665
2666/**
2667 * Does per-VM VT-x initialization.
2668 *
2669 * @returns VBox status code.
2670 * @param pVM Pointer to the VM.
2671 */
2672VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2673{
2674 LogFlowFunc(("pVM=%p\n", pVM));
2675
2676 int rc = hmR0VmxStructsAlloc(pVM);
2677 if (RT_FAILURE(rc))
2678 {
2679 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2680 return rc;
2681 }
2682
2683 return VINF_SUCCESS;
2684}
2685
2686
2687/**
2688 * Does per-VM VT-x termination.
2689 *
2690 * @returns VBox status code.
2691 * @param pVM Pointer to the VM.
2692 */
2693VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2694{
2695 LogFlowFunc(("pVM=%p\n", pVM));
2696
2697#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2698 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2699 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2700#endif
2701 hmR0VmxStructsFree(pVM);
2702 return VINF_SUCCESS;
2703}
2704
2705
2706/**
2707 * Sets up the VM for execution under VT-x.
2708 * This function is only called once per-VM during initialization.
2709 *
2710 * @returns VBox status code.
2711 * @param pVM Pointer to the VM.
2712 */
2713VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2714{
2715 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2716 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2717
2718 LogFlowFunc(("pVM=%p\n", pVM));
2719
2720 /*
2721 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2722 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2723 */
2724 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2725 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2726 || !pVM->hm.s.vmx.pRealModeTSS))
2727 {
2728 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2729 return VERR_INTERNAL_ERROR;
2730 }
2731
2732#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2733 /*
2734 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2735 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2736 */
2737 if ( pVM->hm.s.fAllow64BitGuests
2738 && !HMVMX_IS_64BIT_HOST_MODE())
2739 {
2740 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2741 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2742 }
2743#endif
2744
2745 /* Initialize these always, see hmR3InitFinalizeR0().*/
2746 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2747 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2748
2749 /* Setup the tagged-TLB flush handlers. */
2750 int rc = hmR0VmxSetupTaggedTlb(pVM);
2751 if (RT_FAILURE(rc))
2752 {
2753 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2754 return rc;
2755 }
2756
2757 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2758 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2759#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2760 if ( HMVMX_IS_64BIT_HOST_MODE()
2761 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2762 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2763 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2764 {
2765 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2766 }
2767#endif
2768
2769 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2770 {
2771 PVMCPU pVCpu = &pVM->aCpus[i];
2772 AssertPtr(pVCpu);
2773 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2774
2775 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2776 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2777
2778 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2779 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2780 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2781
2782 /* Set revision dword at the beginning of the VMCS structure. */
2783 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2784
2785 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2786 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2787 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2788 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2789
2790 /* Load this VMCS as the current VMCS. */
2791 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2793 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2794
2795 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2800 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2801 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2802
2803 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2805 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2806
2807 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2809 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2810
2811 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2812 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2813 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2814
2815#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2816 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819#endif
2820
2821 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2822 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2823 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2824 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2825
2826 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2827
2828 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2829 }
2830
2831 return VINF_SUCCESS;
2832}
2833
2834
2835/**
2836 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2837 * the VMCS.
2838 *
2839 * @returns VBox status code.
2840 * @param pVM Pointer to the VM.
2841 * @param pVCpu Pointer to the VMCPU.
2842 */
2843DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2844{
2845 NOREF(pVM); NOREF(pVCpu);
2846
2847 RTCCUINTREG uReg = ASMGetCR0();
2848 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2849 AssertRCReturn(rc, rc);
2850
2851#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2852 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2853 if (HMVMX_IS_64BIT_HOST_MODE())
2854 {
2855 uint64_t uRegCR3 = HMR0Get64bitCR3();
2856 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2857 }
2858 else
2859#endif
2860 {
2861 uReg = ASMGetCR3();
2862 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2863 }
2864 AssertRCReturn(rc, rc);
2865
2866 uReg = ASMGetCR4();
2867 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2868 AssertRCReturn(rc, rc);
2869 return rc;
2870}
2871
2872
2873#if HC_ARCH_BITS == 64
2874/**
2875 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2876 * requirements. See hmR0VmxSaveHostSegmentRegs().
2877 */
2878# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2879 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2880 { \
2881 bool fValidSelector = true; \
2882 if ((selValue) & X86_SEL_LDT) \
2883 { \
2884 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2885 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2886 } \
2887 if (fValidSelector) \
2888 { \
2889 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2890 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2891 } \
2892 (selValue) = 0; \
2893 }
2894#endif
2895
2896
2897/**
2898 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2899 * the host-state area in the VMCS.
2900 *
2901 * @returns VBox status code.
2902 * @param pVM Pointer to the VM.
2903 * @param pVCpu Pointer to the VMCPU.
2904 */
2905DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2906{
2907 int rc = VERR_INTERNAL_ERROR_5;
2908
2909#if HC_ARCH_BITS == 64
2910 /*
2911 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2912 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2913 */
2914 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2915 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2916#endif
2917
2918 /*
2919 * Host DS, ES, FS and GS segment registers.
2920 */
2921#if HC_ARCH_BITS == 64
2922 RTSEL uSelDS = ASMGetDS();
2923 RTSEL uSelES = ASMGetES();
2924 RTSEL uSelFS = ASMGetFS();
2925 RTSEL uSelGS = ASMGetGS();
2926#else
2927 RTSEL uSelDS = 0;
2928 RTSEL uSelES = 0;
2929 RTSEL uSelFS = 0;
2930 RTSEL uSelGS = 0;
2931#endif
2932
2933 /* Recalculate which host-state bits need to be manually restored. */
2934 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2935
2936 /*
2937 * Host CS and SS segment registers.
2938 */
2939#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2940 RTSEL uSelCS;
2941 RTSEL uSelSS;
2942 if (HMVMX_IS_64BIT_HOST_MODE())
2943 {
2944 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2945 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2946 }
2947 else
2948 {
2949 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2950 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2951 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2952 }
2953#else
2954 RTSEL uSelCS = ASMGetCS();
2955 RTSEL uSelSS = ASMGetSS();
2956#endif
2957
2958 /*
2959 * Host TR segment register.
2960 */
2961 RTSEL uSelTR = ASMGetTR();
2962
2963#if HC_ARCH_BITS == 64
2964 /*
2965 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2966 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2967 */
2968 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2969 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2970 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2971 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2972# undef VMXLOCAL_ADJUST_HOST_SEG
2973#endif
2974
2975 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2976 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2977 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2978 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2979 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2980 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2981 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2982 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2983 Assert(uSelCS);
2984 Assert(uSelTR);
2985
2986 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2987#if 0
2988 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2989 Assert(uSelSS != 0);
2990#endif
2991
2992 /* Write these host selector fields into the host-state area in the VMCS. */
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2994 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2995#if HC_ARCH_BITS == 64
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2999 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
3000#endif
3001 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3002
3003 /*
3004 * Host GDTR and IDTR.
3005 */
3006 RTGDTR Gdtr;
3007 RT_ZERO(Gdtr);
3008#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3009 if (HMVMX_IS_64BIT_HOST_MODE())
3010 {
3011 X86XDTR64 Gdtr64;
3012 X86XDTR64 Idtr64;
3013 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3014 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3015 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3016
3017 Gdtr.cbGdt = Gdtr64.cb;
3018 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3019 }
3020 else
3021#endif
3022 {
3023 RTIDTR Idtr;
3024 ASMGetGDTR(&Gdtr);
3025 ASMGetIDTR(&Idtr);
3026 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3027 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3028
3029#if HC_ARCH_BITS == 64
3030 /*
3031 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3032 * maximum limit (0xffff) on every VM-exit.
3033 */
3034 if (Gdtr.cbGdt != 0xffff)
3035 {
3036 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3037 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3038 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3039 }
3040
3041 /*
3042 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3043 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3044 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3045 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3046 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3047 * hosts where we are pretty sure it won't cause trouble.
3048 */
3049# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3050 if (Idtr.cbIdt < 0x0fff)
3051# else
3052 if (Idtr.cbIdt != 0xffff)
3053# endif
3054 {
3055 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3056 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3057 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3058 }
3059#endif
3060 }
3061
3062 /*
3063 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3064 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3065 */
3066 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3067 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3068 VERR_VMX_INVALID_HOST_STATE);
3069
3070 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3071#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3072 if (HMVMX_IS_64BIT_HOST_MODE())
3073 {
3074 /* We need the 64-bit TR base for hybrid darwin. */
3075 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3076 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3077 }
3078 else
3079#endif
3080 {
3081 uintptr_t uTRBase;
3082#if HC_ARCH_BITS == 64
3083 uTRBase = X86DESC64_BASE(pDesc);
3084
3085 /*
3086 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3087 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3088 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3089 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3090 *
3091 * [1] See Intel spec. 3.5 "System Descriptor Types".
3092 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3093 */
3094 Assert(pDesc->System.u4Type == 11);
3095 if ( pDesc->System.u16LimitLow != 0x67
3096 || pDesc->System.u4LimitHigh)
3097 {
3098 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3099 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3100 if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3101 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3102 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3103
3104 /* Store the GDTR here as we need it while restoring TR. */
3105 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3106 }
3107#else
3108 uTRBase = X86DESC_BASE(pDesc);
3109#endif
3110 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3111 }
3112 AssertRCReturn(rc, rc);
3113
3114 /*
3115 * Host FS base and GS base.
3116 */
3117#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3118 if (HMVMX_IS_64BIT_HOST_MODE())
3119 {
3120 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3121 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3122 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3123 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3124
3125# if HC_ARCH_BITS == 64
3126 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3127 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3128 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3129 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3130 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3131# endif
3132 }
3133#endif
3134 return rc;
3135}
3136
3137
3138/**
3139 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3140 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3141 * the host after every successful VM-exit.
3142 *
3143 * @returns VBox status code.
3144 * @param pVM Pointer to the VM.
3145 * @param pVCpu Pointer to the VMCPU.
3146 *
3147 * @remarks No-long-jump zone!!!
3148 */
3149DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3150{
3151 NOREF(pVM);
3152
3153 AssertPtr(pVCpu);
3154 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3155
3156 int rc = VINF_SUCCESS;
3157#if HC_ARCH_BITS == 64
3158 if (pVM->hm.s.fAllow64BitGuests)
3159 hmR0VmxLazySaveHostMsrs(pVCpu);
3160#endif
3161
3162 /*
3163 * Host Sysenter MSRs.
3164 */
3165 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3166 AssertRCReturn(rc, rc);
3167#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3168 if (HMVMX_IS_64BIT_HOST_MODE())
3169 {
3170 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3171 AssertRCReturn(rc, rc);
3172 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3173 }
3174 else
3175 {
3176 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3177 AssertRCReturn(rc, rc);
3178 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3179 }
3180#elif HC_ARCH_BITS == 32
3181 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3182 AssertRCReturn(rc, rc);
3183 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3184#else
3185 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3186 AssertRCReturn(rc, rc);
3187 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3188#endif
3189 AssertRCReturn(rc, rc);
3190
3191 /*
3192 * Host EFER MSR.
3193 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3194 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3195 */
3196 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3197 {
3198 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3199 AssertRCReturn(rc, rc);
3200 }
3201
3202 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3203 * hmR0VmxLoadGuestExitCtls() !! */
3204
3205 return rc;
3206}
3207
3208
3209/**
3210 * Figures out if we need to swap the EFER MSR which is
3211 * particularly expensive.
3212 *
3213 * We check all relevant bits. For now, that's everything
3214 * besides LMA/LME, as these two bits are handled by VM-entry,
3215 * see hmR0VmxLoadGuestExitCtls() and
3216 * hmR0VMxLoadGuestEntryCtls().
3217 *
3218 * @returns true if we need to load guest EFER, false otherwise.
3219 * @param pVCpu Pointer to the VMCPU.
3220 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3221 * out-of-sync. Make sure to update the required fields
3222 * before using them.
3223 *
3224 * @remarks Requires EFER, CR4.
3225 * @remarks No-long-jump zone!!!
3226 */
3227static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3228{
3229#ifdef HMVMX_ALWAYS_SWAP_EFER
3230 return true;
3231#endif
3232
3233#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3234 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3235 if (CPUMIsGuestInLongMode(pVCpu))
3236 return false;
3237#endif
3238
3239 PVM pVM = pVCpu->CTX_SUFF(pVM);
3240 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3241 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3242
3243 /*
3244 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3245 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3246 */
3247 if ( CPUMIsGuestInLongMode(pVCpu)
3248 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3249 {
3250 return true;
3251 }
3252
3253 /*
3254 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3255 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3256 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3257 */
3258 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3259 && (pMixedCtx->cr0 & X86_CR0_PG)
3260 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3261 {
3262 /* Assert that host is PAE capable. */
3263 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3264 return true;
3265 }
3266
3267 /** @todo Check the latest Intel spec. for any other bits,
3268 * like SMEP/SMAP? */
3269 return false;
3270}
3271
3272
3273/**
3274 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3275 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3276 * controls".
3277 *
3278 * @returns VBox status code.
3279 * @param pVCpu Pointer to the VMCPU.
3280 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3281 * out-of-sync. Make sure to update the required fields
3282 * before using them.
3283 *
3284 * @remarks Requires EFER.
3285 * @remarks No-long-jump zone!!!
3286 */
3287DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3288{
3289 int rc = VINF_SUCCESS;
3290 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3291 {
3292 PVM pVM = pVCpu->CTX_SUFF(pVM);
3293 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3294 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3295
3296 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3297 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3298
3299 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3300 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3301 {
3302 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3303 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3304 }
3305 else
3306 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3307
3308 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3309 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3310 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3311 {
3312 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3313 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3314 }
3315
3316 /*
3317 * The following should -not- be set (since we're not in SMM mode):
3318 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3319 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3320 */
3321
3322 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3323 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3324
3325 if ((val & zap) != val)
3326 {
3327 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3328 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3329 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3330 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3331 }
3332
3333 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3334 AssertRCReturn(rc, rc);
3335
3336 pVCpu->hm.s.vmx.u32EntryCtls = val;
3337 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3338 }
3339 return rc;
3340}
3341
3342
3343/**
3344 * Sets up the VM-exit controls in the VMCS.
3345 *
3346 * @returns VBox status code.
3347 * @param pVM Pointer to the VM.
3348 * @param pVCpu Pointer to the VMCPU.
3349 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3350 * out-of-sync. Make sure to update the required fields
3351 * before using them.
3352 *
3353 * @remarks Requires EFER.
3354 */
3355DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3356{
3357 NOREF(pMixedCtx);
3358
3359 int rc = VINF_SUCCESS;
3360 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3361 {
3362 PVM pVM = pVCpu->CTX_SUFF(pVM);
3363 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3364 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3365
3366 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3367 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3368
3369 /*
3370 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3371 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3372 */
3373#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3374 if (HMVMX_IS_64BIT_HOST_MODE())
3375 {
3376 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3377 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3378 }
3379 else
3380 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3381#else
3382 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3383 {
3384 /* The switcher returns to long mode, EFER is managed by the switcher. */
3385 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3386 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3387 }
3388 else
3389 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3390#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3391
3392 /* If the newer VMCS fields for managing EFER exists, use it. */
3393 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3394 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3395 {
3396 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3397 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3398 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3399 }
3400
3401 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3402 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3403
3404 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3405 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3406 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3407
3408 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3409 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3410
3411 if ((val & zap) != val)
3412 {
3413 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3414 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3415 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3416 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3417 }
3418
3419 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3420 AssertRCReturn(rc, rc);
3421
3422 pVCpu->hm.s.vmx.u32ExitCtls = val;
3423 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3424 }
3425 return rc;
3426}
3427
3428
3429/**
3430 * Loads the guest APIC and related state.
3431 *
3432 * @returns VBox status code.
3433 * @param pVM Pointer to the VM.
3434 * @param pVCpu Pointer to the VMCPU.
3435 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3436 * out-of-sync. Make sure to update the required fields
3437 * before using them.
3438 */
3439DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3440{
3441 NOREF(pMixedCtx);
3442
3443 int rc = VINF_SUCCESS;
3444 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3445 {
3446 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3447 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3448 {
3449 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3450
3451 bool fPendingIntr = false;
3452 uint8_t u8Tpr = 0;
3453 uint8_t u8PendingIntr = 0;
3454 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3455 AssertRCReturn(rc, rc);
3456
3457 /*
3458 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3459 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3460 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3461 * the interrupt when we VM-exit for other reasons.
3462 */
3463 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3464 uint32_t u32TprThreshold = 0;
3465 if (fPendingIntr)
3466 {
3467 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3468 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3469 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3470 if (u8PendingPriority <= u8TprPriority)
3471 u32TprThreshold = u8PendingPriority;
3472 else
3473 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3474 }
3475 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3476
3477 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3478 AssertRCReturn(rc, rc);
3479 }
3480
3481 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3482 }
3483 return rc;
3484}
3485
3486
3487/**
3488 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3489 *
3490 * @returns Guest's interruptibility-state.
3491 * @param pVCpu Pointer to the VMCPU.
3492 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3493 * out-of-sync. Make sure to update the required fields
3494 * before using them.
3495 *
3496 * @remarks No-long-jump zone!!!
3497 */
3498DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3499{
3500 /*
3501 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3502 */
3503 uint32_t uIntrState = 0;
3504 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3505 {
3506 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3507 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3508 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3509 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3510 {
3511 if (pMixedCtx->eflags.Bits.u1IF)
3512 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3513 else
3514 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3515 }
3516 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3517 }
3518
3519 /*
3520 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3521 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3522 * setting this would block host-NMIs and IRET will not clear the blocking.
3523 *
3524 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3525 */
3526 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3527 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3528 {
3529 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3530 }
3531
3532 return uIntrState;
3533}
3534
3535
3536/**
3537 * Loads the guest's interruptibility-state into the guest-state area in the
3538 * VMCS.
3539 *
3540 * @returns VBox status code.
3541 * @param pVCpu Pointer to the VMCPU.
3542 * @param uIntrState The interruptibility-state to set.
3543 */
3544static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3545{
3546 NOREF(pVCpu);
3547 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3548 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3549 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3550 AssertRCReturn(rc, rc);
3551 return rc;
3552}
3553
3554
3555/**
3556 * Loads the guest's RIP into the guest-state area in the VMCS.
3557 *
3558 * @returns VBox status code.
3559 * @param pVCpu Pointer to the VMCPU.
3560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3561 * out-of-sync. Make sure to update the required fields
3562 * before using them.
3563 *
3564 * @remarks No-long-jump zone!!!
3565 */
3566static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3567{
3568 int rc = VINF_SUCCESS;
3569 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3570 {
3571 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3572 AssertRCReturn(rc, rc);
3573
3574 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3575 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3576 HMCPU_CF_VALUE(pVCpu)));
3577 }
3578 return rc;
3579}
3580
3581
3582/**
3583 * Loads the guest's RSP into the guest-state area in the VMCS.
3584 *
3585 * @returns VBox status code.
3586 * @param pVCpu Pointer to the VMCPU.
3587 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3588 * out-of-sync. Make sure to update the required fields
3589 * before using them.
3590 *
3591 * @remarks No-long-jump zone!!!
3592 */
3593static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3594{
3595 int rc = VINF_SUCCESS;
3596 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3597 {
3598 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3599 AssertRCReturn(rc, rc);
3600
3601 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3602 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3603 }
3604 return rc;
3605}
3606
3607
3608/**
3609 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3610 *
3611 * @returns VBox status code.
3612 * @param pVCpu Pointer to the VMCPU.
3613 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3614 * out-of-sync. Make sure to update the required fields
3615 * before using them.
3616 *
3617 * @remarks No-long-jump zone!!!
3618 */
3619static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3620{
3621 int rc = VINF_SUCCESS;
3622 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3623 {
3624 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3625 Let us assert it as such and use 32-bit VMWRITE. */
3626 Assert(!(pMixedCtx->rflags.u64 >> 32));
3627 X86EFLAGS Eflags = pMixedCtx->eflags;
3628 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3629 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3630 * These will never be cleared/set, unless some other part of the VMM
3631 * code is buggy - in which case we're better of finding and fixing
3632 * those bugs than hiding them. */
3633 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3634 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3635 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3636 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3637
3638 /*
3639 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3640 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3641 */
3642 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3643 {
3644 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3645 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3646 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3647 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3648 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3649 }
3650
3651 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3652 AssertRCReturn(rc, rc);
3653
3654 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3655 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3656 }
3657 return rc;
3658}
3659
3660
3661/**
3662 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3663 *
3664 * @returns VBox status code.
3665 * @param pVCpu Pointer to the VMCPU.
3666 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3667 * out-of-sync. Make sure to update the required fields
3668 * before using them.
3669 *
3670 * @remarks No-long-jump zone!!!
3671 */
3672DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3673{
3674 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3675 AssertRCReturn(rc, rc);
3676 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3677 AssertRCReturn(rc, rc);
3678 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3679 AssertRCReturn(rc, rc);
3680 return rc;
3681}
3682
3683
3684/**
3685 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3686 * CR0 is partially shared with the host and we have to consider the FPU bits.
3687 *
3688 * @returns VBox status code.
3689 * @param pVM Pointer to the VM.
3690 * @param pVCpu Pointer to the VMCPU.
3691 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3692 * out-of-sync. Make sure to update the required fields
3693 * before using them.
3694 *
3695 * @remarks No-long-jump zone!!!
3696 */
3697static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3698{
3699 /*
3700 * Guest CR0.
3701 * Guest FPU.
3702 */
3703 int rc = VINF_SUCCESS;
3704 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3705 {
3706 Assert(!(pMixedCtx->cr0 >> 32));
3707 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3708 PVM pVM = pVCpu->CTX_SUFF(pVM);
3709
3710 /* The guest's view (read access) of its CR0 is unblemished. */
3711 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3712 AssertRCReturn(rc, rc);
3713 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3714
3715 /* Setup VT-x's view of the guest CR0. */
3716 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3717 if (pVM->hm.s.fNestedPaging)
3718 {
3719 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3720 {
3721 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3722 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3723 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3724 }
3725 else
3726 {
3727 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3728 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3729 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3730 }
3731
3732 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3733 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3734 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3735
3736 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3737 AssertRCReturn(rc, rc);
3738 }
3739 else
3740 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3741
3742 /*
3743 * Guest FPU bits.
3744 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3745 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3746 */
3747 u32GuestCR0 |= X86_CR0_NE;
3748 bool fInterceptNM = false;
3749 if (CPUMIsGuestFPUStateActive(pVCpu))
3750 {
3751 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3752 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3753 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3754 }
3755 else
3756 {
3757 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3758 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3759 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3760 }
3761
3762 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3763 bool fInterceptMF = false;
3764 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3765 fInterceptMF = true;
3766
3767 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3768 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3769 {
3770 Assert(PDMVmmDevHeapIsEnabled(pVM));
3771 Assert(pVM->hm.s.vmx.pRealModeTSS);
3772 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3773 fInterceptNM = true;
3774 fInterceptMF = true;
3775 }
3776 else
3777 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3778
3779 if (fInterceptNM)
3780 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3781 else
3782 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3783
3784 if (fInterceptMF)
3785 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3786 else
3787 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3788
3789 /* Additional intercepts for debugging, define these yourself explicitly. */
3790#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3791 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3792 | RT_BIT(X86_XCPT_BP)
3793 | RT_BIT(X86_XCPT_DB)
3794 | RT_BIT(X86_XCPT_DE)
3795 | RT_BIT(X86_XCPT_NM)
3796 | RT_BIT(X86_XCPT_TS)
3797 | RT_BIT(X86_XCPT_UD)
3798 | RT_BIT(X86_XCPT_NP)
3799 | RT_BIT(X86_XCPT_SS)
3800 | RT_BIT(X86_XCPT_GP)
3801 | RT_BIT(X86_XCPT_PF)
3802 | RT_BIT(X86_XCPT_MF)
3803 ;
3804#elif defined(HMVMX_ALWAYS_TRAP_PF)
3805 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3806#endif
3807
3808 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3809
3810 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3811 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3812 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3813 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3814 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3815 else
3816 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3817
3818 u32GuestCR0 |= uSetCR0;
3819 u32GuestCR0 &= uZapCR0;
3820 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3821
3822 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3823 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3824 AssertRCReturn(rc, rc);
3825 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3826 AssertRCReturn(rc, rc);
3827 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3828 uZapCR0));
3829
3830 /*
3831 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3832 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3833 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3834 */
3835 uint32_t u32CR0Mask = 0;
3836 u32CR0Mask = X86_CR0_PE
3837 | X86_CR0_NE
3838 | X86_CR0_WP
3839 | X86_CR0_PG
3840 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3841 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3842 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3843
3844 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3845 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3846 * and @bugref{6944}. */
3847#if 0
3848 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3849 u32CR0Mask &= ~X86_CR0_PE;
3850#endif
3851 if (pVM->hm.s.fNestedPaging)
3852 u32CR0Mask &= ~X86_CR0_WP;
3853
3854 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3855 if (fInterceptNM)
3856 {
3857 u32CR0Mask |= X86_CR0_TS
3858 | X86_CR0_MP;
3859 }
3860
3861 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3862 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3863 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3864 AssertRCReturn(rc, rc);
3865 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3866
3867 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3868 }
3869 return rc;
3870}
3871
3872
3873/**
3874 * Loads the guest control registers (CR3, CR4) into the guest-state area
3875 * in the VMCS.
3876 *
3877 * @returns VBox status code.
3878 * @param pVM Pointer to the VM.
3879 * @param pVCpu Pointer to the VMCPU.
3880 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3881 * out-of-sync. Make sure to update the required fields
3882 * before using them.
3883 *
3884 * @remarks No-long-jump zone!!!
3885 */
3886static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3887{
3888 int rc = VINF_SUCCESS;
3889 PVM pVM = pVCpu->CTX_SUFF(pVM);
3890
3891 /*
3892 * Guest CR2.
3893 * It's always loaded in the assembler code. Nothing to do here.
3894 */
3895
3896 /*
3897 * Guest CR3.
3898 */
3899 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3900 {
3901 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3902 if (pVM->hm.s.fNestedPaging)
3903 {
3904 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3905
3906 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3907 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3908 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3909 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3910
3911 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3912 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3913 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3914
3915 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3916 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3917 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3918 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3919
3920 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3921 AssertRCReturn(rc, rc);
3922 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3923
3924 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3925 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3926 {
3927 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3928 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3929 {
3930 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3931 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3932 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3933 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3934 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3935 }
3936
3937 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3938 have Unrestricted Execution to handle the guest when it's not using paging. */
3939 GCPhysGuestCR3 = pMixedCtx->cr3;
3940 }
3941 else
3942 {
3943 /*
3944 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3945 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3946 * EPT takes care of translating it to host-physical addresses.
3947 */
3948 RTGCPHYS GCPhys;
3949 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3950 Assert(PDMVmmDevHeapIsEnabled(pVM));
3951
3952 /* We obtain it here every time as the guest could have relocated this PCI region. */
3953 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3954 AssertRCReturn(rc, rc);
3955
3956 GCPhysGuestCR3 = GCPhys;
3957 }
3958
3959 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3960 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3961 }
3962 else
3963 {
3964 /* Non-nested paging case, just use the hypervisor's CR3. */
3965 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3966
3967 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3968 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3969 }
3970 AssertRCReturn(rc, rc);
3971
3972 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3973 }
3974
3975 /*
3976 * Guest CR4.
3977 */
3978 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3979 {
3980 Assert(!(pMixedCtx->cr4 >> 32));
3981 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3982
3983 /* The guest's view of its CR4 is unblemished. */
3984 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3985 AssertRCReturn(rc, rc);
3986 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3987
3988 /* Setup VT-x's view of the guest CR4. */
3989 /*
3990 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3991 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3992 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3993 */
3994 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3995 {
3996 Assert(pVM->hm.s.vmx.pRealModeTSS);
3997 Assert(PDMVmmDevHeapIsEnabled(pVM));
3998 u32GuestCR4 &= ~X86_CR4_VME;
3999 }
4000
4001 if (pVM->hm.s.fNestedPaging)
4002 {
4003 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4004 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4005 {
4006 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4007 u32GuestCR4 |= X86_CR4_PSE;
4008 /* Our identity mapping is a 32-bit page directory. */
4009 u32GuestCR4 &= ~X86_CR4_PAE;
4010 }
4011 /* else use guest CR4.*/
4012 }
4013 else
4014 {
4015 /*
4016 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4017 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4018 */
4019 switch (pVCpu->hm.s.enmShadowMode)
4020 {
4021 case PGMMODE_REAL: /* Real-mode. */
4022 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4023 case PGMMODE_32_BIT: /* 32-bit paging. */
4024 {
4025 u32GuestCR4 &= ~X86_CR4_PAE;
4026 break;
4027 }
4028
4029 case PGMMODE_PAE: /* PAE paging. */
4030 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4031 {
4032 u32GuestCR4 |= X86_CR4_PAE;
4033 break;
4034 }
4035
4036 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4037 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4038#ifdef VBOX_ENABLE_64_BITS_GUESTS
4039 break;
4040#endif
4041 default:
4042 AssertFailed();
4043 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4044 }
4045 }
4046
4047 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4048 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4049 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4050 u32GuestCR4 |= uSetCR4;
4051 u32GuestCR4 &= uZapCR4;
4052
4053 /* Write VT-x's view of the guest CR4 into the VMCS. */
4054 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4055 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4056 AssertRCReturn(rc, rc);
4057
4058 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4059 uint32_t u32CR4Mask = 0;
4060 u32CR4Mask = X86_CR4_VME
4061 | X86_CR4_PAE
4062 | X86_CR4_PGE
4063 | X86_CR4_PSE
4064 | X86_CR4_VMXE;
4065 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4066 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4067 AssertRCReturn(rc, rc);
4068
4069 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4070 }
4071 return rc;
4072}
4073
4074
4075/**
4076 * Loads the guest debug registers into the guest-state area in the VMCS.
4077 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4078 *
4079 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4080 *
4081 * @returns VBox status code.
4082 * @param pVCpu Pointer to the VMCPU.
4083 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4084 * out-of-sync. Make sure to update the required fields
4085 * before using them.
4086 *
4087 * @remarks No-long-jump zone!!!
4088 */
4089static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4090{
4091 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4092 return VINF_SUCCESS;
4093
4094#ifdef VBOX_STRICT
4095 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4096 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4097 {
4098 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4099 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4100 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4101 }
4102#endif
4103
4104 int rc;
4105 PVM pVM = pVCpu->CTX_SUFF(pVM);
4106 bool fInterceptDB = false;
4107 bool fInterceptMovDRx = false;
4108 if ( pVCpu->hm.s.fSingleInstruction
4109 || DBGFIsStepping(pVCpu))
4110 {
4111 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4112 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4113 {
4114 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4115 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4116 AssertRCReturn(rc, rc);
4117 Assert(fInterceptDB == false);
4118 }
4119 else
4120 {
4121 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4122 pVCpu->hm.s.fClearTrapFlag = true;
4123 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4124 fInterceptDB = true;
4125 }
4126 }
4127
4128 if ( fInterceptDB
4129 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4130 {
4131 /*
4132 * Use the combined guest and host DRx values found in the hypervisor
4133 * register set because the debugger has breakpoints active or someone
4134 * is single stepping on the host side without a monitor trap flag.
4135 *
4136 * Note! DBGF expects a clean DR6 state before executing guest code.
4137 */
4138#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4139 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4140 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4141 {
4142 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4143 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4144 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4145 }
4146 else
4147#endif
4148 if (!CPUMIsHyperDebugStateActive(pVCpu))
4149 {
4150 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4151 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4152 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4153 }
4154
4155 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4156 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4157 AssertRCReturn(rc, rc);
4158
4159 pVCpu->hm.s.fUsingHyperDR7 = true;
4160 fInterceptDB = true;
4161 fInterceptMovDRx = true;
4162 }
4163 else
4164 {
4165 /*
4166 * If the guest has enabled debug registers, we need to load them prior to
4167 * executing guest code so they'll trigger at the right time.
4168 */
4169 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4170 {
4171#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4172 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4173 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4174 {
4175 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4176 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4177 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4178 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4179 }
4180 else
4181#endif
4182 if (!CPUMIsGuestDebugStateActive(pVCpu))
4183 {
4184 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4185 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4186 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4187 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4188 }
4189 Assert(!fInterceptDB);
4190 Assert(!fInterceptMovDRx);
4191 }
4192 /*
4193 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4194 * must intercept #DB in order to maintain a correct DR6 guest value.
4195 */
4196#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4197 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4198 && !CPUMIsGuestDebugStateActive(pVCpu))
4199#else
4200 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4201#endif
4202 {
4203 fInterceptMovDRx = true;
4204 fInterceptDB = true;
4205 }
4206
4207 /* Update guest DR7. */
4208 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4209 AssertRCReturn(rc, rc);
4210
4211 pVCpu->hm.s.fUsingHyperDR7 = false;
4212 }
4213
4214 /*
4215 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4216 */
4217 if ( fInterceptDB
4218 || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4219 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4220 else
4221 {
4222#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4223 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4224#endif
4225 }
4226 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4227 AssertRCReturn(rc, rc);
4228
4229 /*
4230 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4231 */
4232 if (fInterceptMovDRx)
4233 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4234 else
4235 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4236 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4237 AssertRCReturn(rc, rc);
4238
4239 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4240 return VINF_SUCCESS;
4241}
4242
4243
4244#ifdef VBOX_STRICT
4245/**
4246 * Strict function to validate segment registers.
4247 *
4248 * @remarks ASSUMES CR0 is up to date.
4249 */
4250static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4251{
4252 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4253 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4254 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4255 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4256 && ( !CPUMIsGuestInRealModeEx(pCtx)
4257 && !CPUMIsGuestInV86ModeEx(pCtx)))
4258 {
4259 /* Protected mode checks */
4260 /* CS */
4261 Assert(pCtx->cs.Attr.n.u1Present);
4262 Assert(!(pCtx->cs.Attr.u & 0xf00));
4263 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4264 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4265 || !(pCtx->cs.Attr.n.u1Granularity));
4266 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4267 || (pCtx->cs.Attr.n.u1Granularity));
4268 /* CS cannot be loaded with NULL in protected mode. */
4269 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4270 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4271 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4272 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4273 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4274 else
4275 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4276 /* SS */
4277 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4278 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4279 if ( !(pCtx->cr0 & X86_CR0_PE)
4280 || pCtx->cs.Attr.n.u4Type == 3)
4281 {
4282 Assert(!pCtx->ss.Attr.n.u2Dpl);
4283 }
4284 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4285 {
4286 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4287 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4288 Assert(pCtx->ss.Attr.n.u1Present);
4289 Assert(!(pCtx->ss.Attr.u & 0xf00));
4290 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4291 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4292 || !(pCtx->ss.Attr.n.u1Granularity));
4293 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4294 || (pCtx->ss.Attr.n.u1Granularity));
4295 }
4296 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4297 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4298 {
4299 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4300 Assert(pCtx->ds.Attr.n.u1Present);
4301 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4302 Assert(!(pCtx->ds.Attr.u & 0xf00));
4303 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4304 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4305 || !(pCtx->ds.Attr.n.u1Granularity));
4306 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4307 || (pCtx->ds.Attr.n.u1Granularity));
4308 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4309 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4310 }
4311 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4312 {
4313 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4314 Assert(pCtx->es.Attr.n.u1Present);
4315 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4316 Assert(!(pCtx->es.Attr.u & 0xf00));
4317 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4318 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4319 || !(pCtx->es.Attr.n.u1Granularity));
4320 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4321 || (pCtx->es.Attr.n.u1Granularity));
4322 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4323 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4324 }
4325 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4326 {
4327 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4328 Assert(pCtx->fs.Attr.n.u1Present);
4329 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4330 Assert(!(pCtx->fs.Attr.u & 0xf00));
4331 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4332 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4333 || !(pCtx->fs.Attr.n.u1Granularity));
4334 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4335 || (pCtx->fs.Attr.n.u1Granularity));
4336 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4337 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4338 }
4339 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4340 {
4341 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4342 Assert(pCtx->gs.Attr.n.u1Present);
4343 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4344 Assert(!(pCtx->gs.Attr.u & 0xf00));
4345 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4346 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4347 || !(pCtx->gs.Attr.n.u1Granularity));
4348 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4349 || (pCtx->gs.Attr.n.u1Granularity));
4350 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4351 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4352 }
4353 /* 64-bit capable CPUs. */
4354# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4355 if (HMVMX_IS_64BIT_HOST_MODE())
4356 {
4357 Assert(!(pCtx->cs.u64Base >> 32));
4358 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4359 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4360 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4361 }
4362# endif
4363 }
4364 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4365 || ( CPUMIsGuestInRealModeEx(pCtx)
4366 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4367 {
4368 /* Real and v86 mode checks. */
4369 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4370 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4371 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4372 {
4373 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4374 }
4375 else
4376 {
4377 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4378 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4379 }
4380
4381 /* CS */
4382 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4383 Assert(pCtx->cs.u32Limit == 0xffff);
4384 Assert(u32CSAttr == 0xf3);
4385 /* SS */
4386 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4387 Assert(pCtx->ss.u32Limit == 0xffff);
4388 Assert(u32SSAttr == 0xf3);
4389 /* DS */
4390 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4391 Assert(pCtx->ds.u32Limit == 0xffff);
4392 Assert(u32DSAttr == 0xf3);
4393 /* ES */
4394 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4395 Assert(pCtx->es.u32Limit == 0xffff);
4396 Assert(u32ESAttr == 0xf3);
4397 /* FS */
4398 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4399 Assert(pCtx->fs.u32Limit == 0xffff);
4400 Assert(u32FSAttr == 0xf3);
4401 /* GS */
4402 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4403 Assert(pCtx->gs.u32Limit == 0xffff);
4404 Assert(u32GSAttr == 0xf3);
4405 /* 64-bit capable CPUs. */
4406# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4407 if (HMVMX_IS_64BIT_HOST_MODE())
4408 {
4409 Assert(!(pCtx->cs.u64Base >> 32));
4410 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4411 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4412 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4413 }
4414# endif
4415 }
4416}
4417#endif /* VBOX_STRICT */
4418
4419
4420/**
4421 * Writes a guest segment register into the guest-state area in the VMCS.
4422 *
4423 * @returns VBox status code.
4424 * @param pVCpu Pointer to the VMCPU.
4425 * @param idxSel Index of the selector in the VMCS.
4426 * @param idxLimit Index of the segment limit in the VMCS.
4427 * @param idxBase Index of the segment base in the VMCS.
4428 * @param idxAccess Index of the access rights of the segment in the VMCS.
4429 * @param pSelReg Pointer to the segment selector.
4430 *
4431 * @remarks No-long-jump zone!!!
4432 */
4433static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4434 uint32_t idxAccess, PCPUMSELREG pSelReg)
4435{
4436 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4437 AssertRCReturn(rc, rc);
4438 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4439 AssertRCReturn(rc, rc);
4440 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4441 AssertRCReturn(rc, rc);
4442
4443 uint32_t u32Access = pSelReg->Attr.u;
4444 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4445 {
4446 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4447 u32Access = 0xf3;
4448 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4449 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4450 }
4451 else
4452 {
4453 /*
4454 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4455 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4456 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4457 * loaded in protected-mode have their attribute as 0.
4458 */
4459 if (!u32Access)
4460 u32Access = X86DESCATTR_UNUSABLE;
4461 }
4462
4463 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4464 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4465 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4466
4467 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4468 AssertRCReturn(rc, rc);
4469 return rc;
4470}
4471
4472
4473/**
4474 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4475 * into the guest-state area in the VMCS.
4476 *
4477 * @returns VBox status code.
4478 * @param pVM Pointer to the VM.
4479 * @param pVCPU Pointer to the VMCPU.
4480 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4481 * out-of-sync. Make sure to update the required fields
4482 * before using them.
4483 *
4484 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4485 * @remarks No-long-jump zone!!!
4486 */
4487static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4488{
4489 int rc = VERR_INTERNAL_ERROR_5;
4490 PVM pVM = pVCpu->CTX_SUFF(pVM);
4491
4492 /*
4493 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4494 */
4495 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4496 {
4497 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4498 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4499 {
4500 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4501 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4502 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4503 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4504 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4505 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4506 }
4507
4508#ifdef VBOX_WITH_REM
4509 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4510 {
4511 Assert(pVM->hm.s.vmx.pRealModeTSS);
4512 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4513 if ( pVCpu->hm.s.vmx.fWasInRealMode
4514 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4515 {
4516 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4517 in real-mode (e.g. OpenBSD 4.0) */
4518 REMFlushTBs(pVM);
4519 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4520 pVCpu->hm.s.vmx.fWasInRealMode = false;
4521 }
4522 }
4523#endif
4524 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4525 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4526 AssertRCReturn(rc, rc);
4527 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4528 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4529 AssertRCReturn(rc, rc);
4530 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4531 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4532 AssertRCReturn(rc, rc);
4533 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4534 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4535 AssertRCReturn(rc, rc);
4536 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4537 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4538 AssertRCReturn(rc, rc);
4539 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4540 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4541 AssertRCReturn(rc, rc);
4542
4543#ifdef VBOX_STRICT
4544 /* Validate. */
4545 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4546#endif
4547
4548 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4549 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4550 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4551 }
4552
4553 /*
4554 * Guest TR.
4555 */
4556 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4557 {
4558 /*
4559 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4560 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4561 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4562 */
4563 uint16_t u16Sel = 0;
4564 uint32_t u32Limit = 0;
4565 uint64_t u64Base = 0;
4566 uint32_t u32AccessRights = 0;
4567
4568 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4569 {
4570 u16Sel = pMixedCtx->tr.Sel;
4571 u32Limit = pMixedCtx->tr.u32Limit;
4572 u64Base = pMixedCtx->tr.u64Base;
4573 u32AccessRights = pMixedCtx->tr.Attr.u;
4574 }
4575 else
4576 {
4577 Assert(pVM->hm.s.vmx.pRealModeTSS);
4578 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4579
4580 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4581 RTGCPHYS GCPhys;
4582 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4583 AssertRCReturn(rc, rc);
4584
4585 X86DESCATTR DescAttr;
4586 DescAttr.u = 0;
4587 DescAttr.n.u1Present = 1;
4588 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4589
4590 u16Sel = 0;
4591 u32Limit = HM_VTX_TSS_SIZE;
4592 u64Base = GCPhys; /* in real-mode phys = virt. */
4593 u32AccessRights = DescAttr.u;
4594 }
4595
4596 /* Validate. */
4597 Assert(!(u16Sel & RT_BIT(2)));
4598 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4599 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4600 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4601 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4602 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4603 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4604 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4605 Assert( (u32Limit & 0xfff) == 0xfff
4606 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4607 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4608 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4609
4610 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4611 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4612 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4613 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4614
4615 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4616 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4617 }
4618
4619 /*
4620 * Guest GDTR.
4621 */
4622 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4623 {
4624 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4625 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4626
4627 /* Validate. */
4628 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4629
4630 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4631 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4632 }
4633
4634 /*
4635 * Guest LDTR.
4636 */
4637 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4638 {
4639 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4640 uint32_t u32Access = 0;
4641 if (!pMixedCtx->ldtr.Attr.u)
4642 u32Access = X86DESCATTR_UNUSABLE;
4643 else
4644 u32Access = pMixedCtx->ldtr.Attr.u;
4645
4646 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4647 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4648 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4649 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4650
4651 /* Validate. */
4652 if (!(u32Access & X86DESCATTR_UNUSABLE))
4653 {
4654 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4655 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4656 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4657 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4658 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4659 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4660 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4661 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4662 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4663 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4664 }
4665
4666 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4667 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4668 }
4669
4670 /*
4671 * Guest IDTR.
4672 */
4673 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4674 {
4675 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4676 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4677
4678 /* Validate. */
4679 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4680
4681 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4682 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4683 }
4684
4685 return VINF_SUCCESS;
4686}
4687
4688
4689/**
4690 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4691 * areas. These MSRs will automatically be loaded to the host CPU on every
4692 * successful VM-entry and stored from the host CPU on every successful VM-exit.
4693 *
4694 * This also creates/updates MSR slots for the host MSRs. The actual host
4695 * MSR values are -not- updated here for performance reasons. See
4696 * hmR0VmxSaveHostMsrs().
4697 *
4698 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4699 *
4700 * @returns VBox status code.
4701 * @param pVCpu Pointer to the VMCPU.
4702 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4703 * out-of-sync. Make sure to update the required fields
4704 * before using them.
4705 *
4706 * @remarks No-long-jump zone!!!
4707 */
4708static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4709{
4710 AssertPtr(pVCpu);
4711 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4712
4713 /*
4714 * MSRs that we use the auto-load/store MSR area in the VMCS.
4715 */
4716 PVM pVM = pVCpu->CTX_SUFF(pVM);
4717 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4718 {
4719 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4720#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4721 if (pVM->hm.s.fAllow64BitGuests)
4722 {
4723 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4724 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4725 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4726 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4727# ifdef DEBUG
4728 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4729 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4730 {
4731 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4732 pMsr->u64Value));
4733 }
4734# endif
4735 }
4736#endif
4737 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4738 }
4739
4740 /*
4741 * Guest Sysenter MSRs.
4742 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4743 * VM-exits on WRMSRs for these MSRs.
4744 */
4745 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4746 {
4747 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4748 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4749 }
4750
4751 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4752 {
4753 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4754 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4755 }
4756
4757 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4758 {
4759 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4760 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4761 }
4762
4763 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4764 {
4765 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4766 {
4767 /*
4768 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4769 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4770 */
4771 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4772 {
4773 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4774 AssertRCReturn(rc,rc);
4775 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4776 }
4777 else
4778 {
4779 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4780 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4781 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4782 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4783 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4784 }
4785 }
4786 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4787 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4788 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4789 }
4790
4791 return VINF_SUCCESS;
4792}
4793
4794
4795/**
4796 * Loads the guest activity state into the guest-state area in the VMCS.
4797 *
4798 * @returns VBox status code.
4799 * @param pVCpu Pointer to the VMCPU.
4800 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4801 * out-of-sync. Make sure to update the required fields
4802 * before using them.
4803 *
4804 * @remarks No-long-jump zone!!!
4805 */
4806static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4807{
4808 NOREF(pCtx);
4809 /** @todo See if we can make use of other states, e.g.
4810 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4811 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4812 {
4813 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4814 AssertRCReturn(rc, rc);
4815
4816 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4817 }
4818 return VINF_SUCCESS;
4819}
4820
4821
4822/**
4823 * Sets up the appropriate function to run guest code.
4824 *
4825 * @returns VBox status code.
4826 * @param pVCpu Pointer to the VMCPU.
4827 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4828 * out-of-sync. Make sure to update the required fields
4829 * before using them.
4830 *
4831 * @remarks No-long-jump zone!!!
4832 */
4833static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4834{
4835 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4836 {
4837#ifndef VBOX_ENABLE_64_BITS_GUESTS
4838 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4839#endif
4840 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4841#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4842 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4843 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4844 {
4845 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4846 {
4847 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4848 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4849 | HM_CHANGED_VMX_EXIT_CTLS
4850 | HM_CHANGED_VMX_ENTRY_CTLS
4851 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4852 }
4853 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4854 }
4855#else
4856 /* 64-bit host or hybrid host. */
4857 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4858#endif
4859 }
4860 else
4861 {
4862 /* Guest is not in long mode, use the 32-bit handler. */
4863#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4864 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4865 {
4866 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4867 {
4868 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4869 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4870 | HM_CHANGED_VMX_EXIT_CTLS
4871 | HM_CHANGED_VMX_ENTRY_CTLS
4872 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4873 }
4874 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4875 }
4876#else
4877 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4878#endif
4879 }
4880 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4881 return VINF_SUCCESS;
4882}
4883
4884
4885/**
4886 * Wrapper for running the guest code in VT-x.
4887 *
4888 * @returns VBox strict status code.
4889 * @param pVM Pointer to the VM.
4890 * @param pVCpu Pointer to the VMCPU.
4891 * @param pCtx Pointer to the guest-CPU context.
4892 *
4893 * @remarks No-long-jump zone!!!
4894 */
4895DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4896{
4897 /*
4898 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4899 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4900 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4901 */
4902 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4903 /** @todo Add stats for resume vs launch. */
4904#ifdef VBOX_WITH_KERNEL_USING_XMM
4905 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4906#else
4907 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4908#endif
4909}
4910
4911
4912/**
4913 * Reports world-switch error and dumps some useful debug info.
4914 *
4915 * @param pVM Pointer to the VM.
4916 * @param pVCpu Pointer to the VMCPU.
4917 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4918 * @param pCtx Pointer to the guest-CPU context.
4919 * @param pVmxTransient Pointer to the VMX transient structure (only
4920 * exitReason updated).
4921 */
4922static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4923{
4924 Assert(pVM);
4925 Assert(pVCpu);
4926 Assert(pCtx);
4927 Assert(pVmxTransient);
4928 HMVMX_ASSERT_PREEMPT_SAFE();
4929
4930 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4931 switch (rcVMRun)
4932 {
4933 case VERR_VMX_INVALID_VMXON_PTR:
4934 AssertFailed();
4935 break;
4936 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4937 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4938 {
4939 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4940 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4941 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4942 AssertRC(rc);
4943
4944 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4945 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4946 Cannot do it here as we may have been long preempted. */
4947
4948#ifdef VBOX_STRICT
4949 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4950 pVmxTransient->uExitReason));
4951 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4952 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4953 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4954 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4955 else
4956 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4957 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4958 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4959
4960 /* VMX control bits. */
4961 uint32_t u32Val;
4962 uint64_t u64Val;
4963 HMVMXHCUINTREG uHCReg;
4964 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4965 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4970 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4971 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4974 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4975 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4976 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4977 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4978 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4979 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4980 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4981 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4982 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4983 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4984 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4985 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4986 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4987 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4992 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4993 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4994 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4995 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4996 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4997 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4998 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4999 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5000 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5001 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5002 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5003 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5004 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5005 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5006
5007 /* Guest bits. */
5008 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5009 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5010 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5011 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5012 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5013 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5014 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5015 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5016
5017 /* Host bits. */
5018 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5019 Log4(("Host CR0 %#RHr\n", uHCReg));
5020 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5021 Log4(("Host CR3 %#RHr\n", uHCReg));
5022 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5023 Log4(("Host CR4 %#RHr\n", uHCReg));
5024
5025 RTGDTR HostGdtr;
5026 PCX86DESCHC pDesc;
5027 ASMGetGDTR(&HostGdtr);
5028 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5029 Log4(("Host CS %#08x\n", u32Val));
5030 if (u32Val < HostGdtr.cbGdt)
5031 {
5032 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5033 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5034 }
5035
5036 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5037 Log4(("Host DS %#08x\n", u32Val));
5038 if (u32Val < HostGdtr.cbGdt)
5039 {
5040 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5041 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5042 }
5043
5044 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5045 Log4(("Host ES %#08x\n", u32Val));
5046 if (u32Val < HostGdtr.cbGdt)
5047 {
5048 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5049 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5050 }
5051
5052 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5053 Log4(("Host FS %#08x\n", u32Val));
5054 if (u32Val < HostGdtr.cbGdt)
5055 {
5056 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5057 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5058 }
5059
5060 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5061 Log4(("Host GS %#08x\n", u32Val));
5062 if (u32Val < HostGdtr.cbGdt)
5063 {
5064 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5065 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5066 }
5067
5068 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5069 Log4(("Host SS %#08x\n", u32Val));
5070 if (u32Val < HostGdtr.cbGdt)
5071 {
5072 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5073 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5074 }
5075
5076 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5077 Log4(("Host TR %#08x\n", u32Val));
5078 if (u32Val < HostGdtr.cbGdt)
5079 {
5080 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5081 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5082 }
5083
5084 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5085 Log4(("Host TR Base %#RHv\n", uHCReg));
5086 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5087 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5088 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5089 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5090 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5091 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5092 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5093 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5095 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5096 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5097 Log4(("Host RSP %#RHv\n", uHCReg));
5098 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5099 Log4(("Host RIP %#RHv\n", uHCReg));
5100# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5101 if (HMVMX_IS_64BIT_HOST_MODE())
5102 {
5103 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5104 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5105 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5106 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5107 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5108 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5109 }
5110# endif
5111#endif /* VBOX_STRICT */
5112 break;
5113 }
5114
5115 default:
5116 /* Impossible */
5117 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5118 break;
5119 }
5120 NOREF(pVM); NOREF(pCtx);
5121}
5122
5123
5124#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5125#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5126# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5127#endif
5128#ifdef VBOX_STRICT
5129static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5130{
5131 switch (idxField)
5132 {
5133 case VMX_VMCS_GUEST_RIP:
5134 case VMX_VMCS_GUEST_RSP:
5135 case VMX_VMCS_GUEST_SYSENTER_EIP:
5136 case VMX_VMCS_GUEST_SYSENTER_ESP:
5137 case VMX_VMCS_GUEST_GDTR_BASE:
5138 case VMX_VMCS_GUEST_IDTR_BASE:
5139 case VMX_VMCS_GUEST_CS_BASE:
5140 case VMX_VMCS_GUEST_DS_BASE:
5141 case VMX_VMCS_GUEST_ES_BASE:
5142 case VMX_VMCS_GUEST_FS_BASE:
5143 case VMX_VMCS_GUEST_GS_BASE:
5144 case VMX_VMCS_GUEST_SS_BASE:
5145 case VMX_VMCS_GUEST_LDTR_BASE:
5146 case VMX_VMCS_GUEST_TR_BASE:
5147 case VMX_VMCS_GUEST_CR3:
5148 return true;
5149 }
5150 return false;
5151}
5152
5153static bool hmR0VmxIsValidReadField(uint32_t idxField)
5154{
5155 switch (idxField)
5156 {
5157 /* Read-only fields. */
5158 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5159 return true;
5160 }
5161 /* Remaining readable fields should also be writable. */
5162 return hmR0VmxIsValidWriteField(idxField);
5163}
5164#endif /* VBOX_STRICT */
5165
5166
5167/**
5168 * Executes the specified handler in 64-bit mode.
5169 *
5170 * @returns VBox status code.
5171 * @param pVM Pointer to the VM.
5172 * @param pVCpu Pointer to the VMCPU.
5173 * @param pCtx Pointer to the guest CPU context.
5174 * @param enmOp The operation to perform.
5175 * @param cbParam Number of parameters.
5176 * @param paParam Array of 32-bit parameters.
5177 */
5178VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5179 uint32_t *paParam)
5180{
5181 int rc, rc2;
5182 PHMGLOBALCPUINFO pCpu;
5183 RTHCPHYS HCPhysCpuPage;
5184 RTCCUINTREG uOldEflags;
5185
5186 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5187 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5188 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5189 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5190
5191#ifdef VBOX_STRICT
5192 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5193 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5194
5195 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5196 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5197#endif
5198
5199 /* Disable interrupts. */
5200 uOldEflags = ASMIntDisableFlags();
5201
5202#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5203 RTCPUID idHostCpu = RTMpCpuId();
5204 CPUMR0SetLApic(pVCpu, idHostCpu);
5205#endif
5206
5207 pCpu = HMR0GetCurrentCpu();
5208 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5209
5210 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5211 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5212
5213 /* Leave VMX Root Mode. */
5214 VMXDisable();
5215
5216 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5217
5218 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5219 CPUMSetHyperEIP(pVCpu, enmOp);
5220 for (int i = (int)cbParam - 1; i >= 0; i--)
5221 CPUMPushHyper(pVCpu, paParam[i]);
5222
5223 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5224
5225 /* Call the switcher. */
5226 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5227 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5228
5229 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5230 /* Make sure the VMX instructions don't cause #UD faults. */
5231 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5232
5233 /* Re-enter VMX Root Mode */
5234 rc2 = VMXEnable(HCPhysCpuPage);
5235 if (RT_FAILURE(rc2))
5236 {
5237 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5238 ASMSetFlags(uOldEflags);
5239 return rc2;
5240 }
5241
5242 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5243 AssertRC(rc2);
5244 Assert(!(ASMGetFlags() & X86_EFL_IF));
5245 ASMSetFlags(uOldEflags);
5246 return rc;
5247}
5248
5249
5250/**
5251 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5252 * supporting 64-bit guests.
5253 *
5254 * @returns VBox status code.
5255 * @param fResume Whether to VMLAUNCH or VMRESUME.
5256 * @param pCtx Pointer to the guest-CPU context.
5257 * @param pCache Pointer to the VMCS cache.
5258 * @param pVM Pointer to the VM.
5259 * @param pVCpu Pointer to the VMCPU.
5260 */
5261DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5262{
5263 uint32_t aParam[6];
5264 PHMGLOBALCPUINFO pCpu = NULL;
5265 RTHCPHYS HCPhysCpuPage = 0;
5266 int rc = VERR_INTERNAL_ERROR_5;
5267
5268 pCpu = HMR0GetCurrentCpu();
5269 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5270
5271#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5272 pCache->uPos = 1;
5273 pCache->interPD = PGMGetInterPaeCR3(pVM);
5274 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5275#endif
5276
5277#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5278 pCache->TestIn.HCPhysCpuPage = 0;
5279 pCache->TestIn.HCPhysVmcs = 0;
5280 pCache->TestIn.pCache = 0;
5281 pCache->TestOut.HCPhysVmcs = 0;
5282 pCache->TestOut.pCache = 0;
5283 pCache->TestOut.pCtx = 0;
5284 pCache->TestOut.eflags = 0;
5285#endif
5286
5287 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5288 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5289 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5290 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5291 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5292 aParam[5] = 0;
5293
5294#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5295 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5296 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5297#endif
5298 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5299
5300#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5301 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5302 Assert(pCtx->dr[4] == 10);
5303 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5304#endif
5305
5306#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5307 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5308 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5309 pVCpu->hm.s.vmx.HCPhysVmcs));
5310 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5311 pCache->TestOut.HCPhysVmcs));
5312 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5313 pCache->TestOut.pCache));
5314 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5315 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5316 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5317 pCache->TestOut.pCtx));
5318 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5319#endif
5320 return rc;
5321}
5322
5323
5324/**
5325 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5326 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5327 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5328 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5329 *
5330 * @returns VBox status code.
5331 * @param pVM Pointer to the VM.
5332 * @param pVCpu Pointer to the VMCPU.
5333 */
5334static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5335{
5336#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5337{ \
5338 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5339 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5340 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5341 ++cReadFields; \
5342}
5343
5344 AssertPtr(pVM);
5345 AssertPtr(pVCpu);
5346 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5347 uint32_t cReadFields = 0;
5348
5349 /*
5350 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5351 * and serve to indicate exceptions to the rules.
5352 */
5353
5354 /* Guest-natural selector base fields. */
5355#if 0
5356 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5357 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5359#endif
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5368 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5371 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5372#if 0
5373 /* Unused natural width guest-state fields. */
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5376#endif
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5379
5380 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5381#if 0
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5389 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5391#endif
5392
5393 /* Natural width guest-state fields. */
5394 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5395#if 0
5396 /* Currently unused field. */
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5398#endif
5399
5400 if (pVM->hm.s.fNestedPaging)
5401 {
5402 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5403 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5404 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5405 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5406 }
5407 else
5408 {
5409 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5410 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5411 }
5412
5413#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5414 return VINF_SUCCESS;
5415}
5416
5417
5418/**
5419 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5420 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5421 * darwin, running 64-bit guests).
5422 *
5423 * @returns VBox status code.
5424 * @param pVCpu Pointer to the VMCPU.
5425 * @param idxField The VMCS field encoding.
5426 * @param u64Val 16, 32 or 64-bit value.
5427 */
5428VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5429{
5430 int rc;
5431 switch (idxField)
5432 {
5433 /*
5434 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5435 */
5436 /* 64-bit Control fields. */
5437 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5438 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5439 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5440 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5441 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5442 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5443 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5444 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5445 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5446 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5447 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5448 case VMX_VMCS64_CTRL_EPTP_FULL:
5449 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5450 /* 64-bit Guest-state fields. */
5451 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5452 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5453 case VMX_VMCS64_GUEST_PAT_FULL:
5454 case VMX_VMCS64_GUEST_EFER_FULL:
5455 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5456 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5457 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5458 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5459 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5460 /* 64-bit Host-state fields. */
5461 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5462 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5463 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5464 {
5465 rc = VMXWriteVmcs32(idxField, u64Val);
5466 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5467 break;
5468 }
5469
5470 /*
5471 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5472 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5473 */
5474 /* Natural-width Guest-state fields. */
5475 case VMX_VMCS_GUEST_CR3:
5476 case VMX_VMCS_GUEST_ES_BASE:
5477 case VMX_VMCS_GUEST_CS_BASE:
5478 case VMX_VMCS_GUEST_SS_BASE:
5479 case VMX_VMCS_GUEST_DS_BASE:
5480 case VMX_VMCS_GUEST_FS_BASE:
5481 case VMX_VMCS_GUEST_GS_BASE:
5482 case VMX_VMCS_GUEST_LDTR_BASE:
5483 case VMX_VMCS_GUEST_TR_BASE:
5484 case VMX_VMCS_GUEST_GDTR_BASE:
5485 case VMX_VMCS_GUEST_IDTR_BASE:
5486 case VMX_VMCS_GUEST_RSP:
5487 case VMX_VMCS_GUEST_RIP:
5488 case VMX_VMCS_GUEST_SYSENTER_ESP:
5489 case VMX_VMCS_GUEST_SYSENTER_EIP:
5490 {
5491 if (!(u64Val >> 32))
5492 {
5493 /* If this field is 64-bit, VT-x will zero out the top bits. */
5494 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5495 }
5496 else
5497 {
5498 /* Assert that only the 32->64 switcher case should ever come here. */
5499 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5500 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5501 }
5502 break;
5503 }
5504
5505 default:
5506 {
5507 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5508 rc = VERR_INVALID_PARAMETER;
5509 break;
5510 }
5511 }
5512 AssertRCReturn(rc, rc);
5513 return rc;
5514}
5515
5516
5517/**
5518 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5519 * hosts (except darwin) for 64-bit guests.
5520 *
5521 * @param pVCpu Pointer to the VMCPU.
5522 * @param idxField The VMCS field encoding.
5523 * @param u64Val 16, 32 or 64-bit value.
5524 */
5525VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5526{
5527 AssertPtr(pVCpu);
5528 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5529
5530 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5531 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5532
5533 /* Make sure there are no duplicates. */
5534 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5535 {
5536 if (pCache->Write.aField[i] == idxField)
5537 {
5538 pCache->Write.aFieldVal[i] = u64Val;
5539 return VINF_SUCCESS;
5540 }
5541 }
5542
5543 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5544 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5545 pCache->Write.cValidEntries++;
5546 return VINF_SUCCESS;
5547}
5548
5549/* Enable later when the assembly code uses these as callbacks. */
5550#if 0
5551/*
5552 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5553 *
5554 * @param pVCpu Pointer to the VMCPU.
5555 * @param pCache Pointer to the VMCS cache.
5556 *
5557 * @remarks No-long-jump zone!!!
5558 */
5559VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5560{
5561 AssertPtr(pCache);
5562 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5563 {
5564 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5565 AssertRC(rc);
5566 }
5567 pCache->Write.cValidEntries = 0;
5568}
5569
5570
5571/**
5572 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5573 *
5574 * @param pVCpu Pointer to the VMCPU.
5575 * @param pCache Pointer to the VMCS cache.
5576 *
5577 * @remarks No-long-jump zone!!!
5578 */
5579VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5580{
5581 AssertPtr(pCache);
5582 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5583 {
5584 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5585 AssertRC(rc);
5586 }
5587}
5588#endif
5589#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5590
5591
5592/**
5593 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5594 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5595 * timer.
5596 *
5597 * @returns VBox status code.
5598 * @param pVCpu Pointer to the VMCPU.
5599 *
5600 * @remarks No-long-jump zone!!!
5601 */
5602static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5603{
5604 int rc = VERR_INTERNAL_ERROR_5;
5605 bool fOffsettedTsc = false;
5606 bool fParavirtTsc = false;
5607 PVM pVM = pVCpu->CTX_SUFF(pVM);
5608 if (pVM->hm.s.vmx.fUsePreemptTimer)
5609 {
5610 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fOffsettedTsc,
5611 &fParavirtTsc);
5612
5613 /* Make sure the returned values have sane upper and lower boundaries. */
5614 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5615 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5616 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5617 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5618
5619 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5620 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5621 }
5622 else
5623 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5624
5625 /** @todo later optimize this to be done elsewhere and not before every
5626 * VM-entry. */
5627 if (fParavirtTsc)
5628 {
5629 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5630 AssertRC(rc);
5631 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5632 }
5633
5634 if (fOffsettedTsc)
5635 {
5636 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5637 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5638
5639 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5640 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5641 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5642 }
5643 else
5644 {
5645 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5646 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5647 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5648 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5649 }
5650}
5651
5652
5653/**
5654 * Determines if an exception is a contributory exception.
5655 *
5656 * Contributory exceptions are ones which can cause double-faults unless the
5657 * original exception was a benign exception. Page-fault is intentionally not
5658 * included here as it's a conditional contributory exception.
5659 *
5660 * @returns true if the exception is contributory, false otherwise.
5661 * @param uVector The exception vector.
5662 */
5663DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5664{
5665 switch (uVector)
5666 {
5667 case X86_XCPT_GP:
5668 case X86_XCPT_SS:
5669 case X86_XCPT_NP:
5670 case X86_XCPT_TS:
5671 case X86_XCPT_DE:
5672 return true;
5673 default:
5674 break;
5675 }
5676 return false;
5677}
5678
5679
5680/**
5681 * Sets an event as a pending event to be injected into the guest.
5682 *
5683 * @param pVCpu Pointer to the VMCPU.
5684 * @param u32IntInfo The VM-entry interruption-information field.
5685 * @param cbInstr The VM-entry instruction length in bytes (for software
5686 * interrupts, exceptions and privileged software
5687 * exceptions).
5688 * @param u32ErrCode The VM-entry exception error code.
5689 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5690 * page-fault.
5691 *
5692 * @remarks Statistics counter assumes this is a guest event being injected or
5693 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5694 * always incremented.
5695 */
5696DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5697 RTGCUINTPTR GCPtrFaultAddress)
5698{
5699 Assert(!pVCpu->hm.s.Event.fPending);
5700 pVCpu->hm.s.Event.fPending = true;
5701 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5702 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5703 pVCpu->hm.s.Event.cbInstr = cbInstr;
5704 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5705
5706 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5707}
5708
5709
5710/**
5711 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5712 *
5713 * @param pVCpu Pointer to the VMCPU.
5714 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5715 * out-of-sync. Make sure to update the required fields
5716 * before using them.
5717 */
5718DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5719{
5720 NOREF(pMixedCtx);
5721 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5722 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5723 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5724 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5725}
5726
5727
5728/**
5729 * Handle a condition that occurred while delivering an event through the guest
5730 * IDT.
5731 *
5732 * @returns VBox status code (informational error codes included).
5733 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5734 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5735 * continue execution of the guest which will delivery the #DF.
5736 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5737 *
5738 * @param pVCpu Pointer to the VMCPU.
5739 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5740 * out-of-sync. Make sure to update the required fields
5741 * before using them.
5742 * @param pVmxTransient Pointer to the VMX transient structure.
5743 *
5744 * @remarks No-long-jump zone!!!
5745 */
5746static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5747{
5748 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5749
5750 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5751 AssertRCReturn(rc, rc);
5752 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5753 AssertRCReturn(rc, rc);
5754
5755 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5756 {
5757 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5758 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5759
5760 typedef enum
5761 {
5762 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5763 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5764 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5765 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5766 } VMXREFLECTXCPT;
5767
5768 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5769 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5770 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5771 {
5772 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5773 {
5774 enmReflect = VMXREFLECTXCPT_XCPT;
5775#ifdef VBOX_STRICT
5776 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5777 && uExitVector == X86_XCPT_PF)
5778 {
5779 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5780 }
5781#endif
5782 if ( uExitVector == X86_XCPT_PF
5783 && uIdtVector == X86_XCPT_PF)
5784 {
5785 pVmxTransient->fVectoringDoublePF = true;
5786 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5787 }
5788 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5789 && hmR0VmxIsContributoryXcpt(uExitVector)
5790 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5791 || uIdtVector == X86_XCPT_PF))
5792 {
5793 enmReflect = VMXREFLECTXCPT_DF;
5794 }
5795 else if (uIdtVector == X86_XCPT_DF)
5796 enmReflect = VMXREFLECTXCPT_TF;
5797 }
5798 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5799 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5800 {
5801 /*
5802 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5803 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5804 */
5805 enmReflect = VMXREFLECTXCPT_XCPT;
5806
5807 if (uExitVector == X86_XCPT_PF)
5808 {
5809 pVmxTransient->fVectoringPF = true;
5810 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5811 }
5812 }
5813 }
5814 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5815 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5816 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5817 {
5818 /*
5819 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5820 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5821 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5822 */
5823 enmReflect = VMXREFLECTXCPT_XCPT;
5824 }
5825
5826 /*
5827 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5828 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5829 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5830 *
5831 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5832 */
5833 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5834 && enmReflect == VMXREFLECTXCPT_XCPT
5835 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5836 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5837 {
5838 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5839 }
5840
5841 switch (enmReflect)
5842 {
5843 case VMXREFLECTXCPT_XCPT:
5844 {
5845 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5846 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5847 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5848
5849 uint32_t u32ErrCode = 0;
5850 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5851 {
5852 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5853 AssertRCReturn(rc, rc);
5854 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5855 }
5856
5857 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5858 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5859 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5860 rc = VINF_SUCCESS;
5861 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5862 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5863
5864 break;
5865 }
5866
5867 case VMXREFLECTXCPT_DF:
5868 {
5869 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5870 rc = VINF_HM_DOUBLE_FAULT;
5871 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5872 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5873
5874 break;
5875 }
5876
5877 case VMXREFLECTXCPT_TF:
5878 {
5879 rc = VINF_EM_RESET;
5880 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5881 uExitVector));
5882 break;
5883 }
5884
5885 default:
5886 Assert(rc == VINF_SUCCESS);
5887 break;
5888 }
5889 }
5890 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5891 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5892 && uExitVector != X86_XCPT_DF
5893 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5894 {
5895 /*
5896 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5897 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5898 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5899 */
5900 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5901 {
5902 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5903 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5904 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5905 }
5906 }
5907
5908 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5909 return rc;
5910}
5911
5912
5913/**
5914 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5915 *
5916 * @returns VBox status code.
5917 * @param pVCpu Pointer to the VMCPU.
5918 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5919 * out-of-sync. Make sure to update the required fields
5920 * before using them.
5921 *
5922 * @remarks No-long-jump zone!!!
5923 */
5924static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5925{
5926 NOREF(pMixedCtx);
5927
5928 /*
5929 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5930 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5931 */
5932 VMMRZCallRing3Disable(pVCpu);
5933 HM_DISABLE_PREEMPT_IF_NEEDED();
5934
5935 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5936 {
5937 uint32_t uVal = 0;
5938 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5939 AssertRCReturn(rc, rc);
5940
5941 uint32_t uShadow = 0;
5942 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5943 AssertRCReturn(rc, rc);
5944
5945 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5946 CPUMSetGuestCR0(pVCpu, uVal);
5947 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5948 }
5949
5950 HM_RESTORE_PREEMPT_IF_NEEDED();
5951 VMMRZCallRing3Enable(pVCpu);
5952 return VINF_SUCCESS;
5953}
5954
5955
5956/**
5957 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5958 *
5959 * @returns VBox status code.
5960 * @param pVCpu Pointer to the VMCPU.
5961 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5962 * out-of-sync. Make sure to update the required fields
5963 * before using them.
5964 *
5965 * @remarks No-long-jump zone!!!
5966 */
5967static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5968{
5969 NOREF(pMixedCtx);
5970
5971 int rc = VINF_SUCCESS;
5972 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5973 {
5974 uint32_t uVal = 0;
5975 uint32_t uShadow = 0;
5976 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5977 AssertRCReturn(rc, rc);
5978 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5979 AssertRCReturn(rc, rc);
5980
5981 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5982 CPUMSetGuestCR4(pVCpu, uVal);
5983 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5984 }
5985 return rc;
5986}
5987
5988
5989/**
5990 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5991 *
5992 * @returns VBox status code.
5993 * @param pVCpu Pointer to the VMCPU.
5994 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5995 * out-of-sync. Make sure to update the required fields
5996 * before using them.
5997 *
5998 * @remarks No-long-jump zone!!!
5999 */
6000static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6001{
6002 int rc = VINF_SUCCESS;
6003 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6004 {
6005 uint64_t u64Val = 0;
6006 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6007 AssertRCReturn(rc, rc);
6008
6009 pMixedCtx->rip = u64Val;
6010 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6011 }
6012 return rc;
6013}
6014
6015
6016/**
6017 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6018 *
6019 * @returns VBox status code.
6020 * @param pVCpu Pointer to the VMCPU.
6021 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6022 * out-of-sync. Make sure to update the required fields
6023 * before using them.
6024 *
6025 * @remarks No-long-jump zone!!!
6026 */
6027static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6028{
6029 int rc = VINF_SUCCESS;
6030 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6031 {
6032 uint64_t u64Val = 0;
6033 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6034 AssertRCReturn(rc, rc);
6035
6036 pMixedCtx->rsp = u64Val;
6037 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6038 }
6039 return rc;
6040}
6041
6042
6043/**
6044 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6045 *
6046 * @returns VBox status code.
6047 * @param pVCpu Pointer to the VMCPU.
6048 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6049 * out-of-sync. Make sure to update the required fields
6050 * before using them.
6051 *
6052 * @remarks No-long-jump zone!!!
6053 */
6054static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6055{
6056 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6057 {
6058 uint32_t uVal = 0;
6059 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6060 AssertRCReturn(rc, rc);
6061
6062 pMixedCtx->eflags.u32 = uVal;
6063 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6064 {
6065 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6066 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6067
6068 pMixedCtx->eflags.Bits.u1VM = 0;
6069 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6070 }
6071
6072 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6073 }
6074 return VINF_SUCCESS;
6075}
6076
6077
6078/**
6079 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6080 * guest-CPU context.
6081 */
6082DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6083{
6084 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6085 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6086 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6087 return rc;
6088}
6089
6090
6091/**
6092 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6093 * from the guest-state area in the VMCS.
6094 *
6095 * @param pVCpu Pointer to the VMCPU.
6096 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6097 * out-of-sync. Make sure to update the required fields
6098 * before using them.
6099 *
6100 * @remarks No-long-jump zone!!!
6101 */
6102static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6103{
6104 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6105 {
6106 uint32_t uIntrState = 0;
6107 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6108 AssertRC(rc);
6109
6110 if (!uIntrState)
6111 {
6112 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6113 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6114
6115 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6116 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6117 }
6118 else
6119 {
6120 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6121 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6122 {
6123 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6124 AssertRC(rc);
6125 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6126 AssertRC(rc);
6127
6128 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6129 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6130 }
6131 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6132 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6133
6134 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6135 {
6136 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6137 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6138 }
6139 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6140 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6141 }
6142
6143 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6144 }
6145}
6146
6147
6148/**
6149 * Saves the guest's activity state.
6150 *
6151 * @returns VBox status code.
6152 * @param pVCpu Pointer to the VMCPU.
6153 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6154 * out-of-sync. Make sure to update the required fields
6155 * before using them.
6156 *
6157 * @remarks No-long-jump zone!!!
6158 */
6159static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6160{
6161 NOREF(pMixedCtx);
6162 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6163 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6164 return VINF_SUCCESS;
6165}
6166
6167
6168/**
6169 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6170 * the current VMCS into the guest-CPU context.
6171 *
6172 * @returns VBox status code.
6173 * @param pVCpu Pointer to the VMCPU.
6174 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6175 * out-of-sync. Make sure to update the required fields
6176 * before using them.
6177 *
6178 * @remarks No-long-jump zone!!!
6179 */
6180static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6181{
6182 int rc = VINF_SUCCESS;
6183 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6184 {
6185 uint32_t u32Val = 0;
6186 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6187 pMixedCtx->SysEnter.cs = u32Val;
6188 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6189 }
6190
6191 uint64_t u64Val = 0;
6192 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6193 {
6194 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6195 pMixedCtx->SysEnter.eip = u64Val;
6196 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6197 }
6198 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6199 {
6200 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6201 pMixedCtx->SysEnter.esp = u64Val;
6202 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6203 }
6204 return rc;
6205}
6206
6207
6208/**
6209 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6210 * the CPU back into the guest-CPU context.
6211 *
6212 * @returns VBox status code.
6213 * @param pVCpu Pointer to the VMCPU.
6214 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6215 * out-of-sync. Make sure to update the required fields
6216 * before using them.
6217 *
6218 * @remarks No-long-jump zone!!!
6219 */
6220static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6221{
6222#if HC_ARCH_BITS == 64
6223 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6224 {
6225 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6226 VMMRZCallRing3Disable(pVCpu);
6227 HM_DISABLE_PREEMPT_IF_NEEDED();
6228
6229 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6230 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6231 {
6232 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6233 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6234 }
6235
6236 HM_RESTORE_PREEMPT_IF_NEEDED();
6237 VMMRZCallRing3Enable(pVCpu);
6238 }
6239 else
6240 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6241#else
6242 NOREF(pMixedCtx);
6243 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6244#endif
6245
6246 return VINF_SUCCESS;
6247}
6248
6249
6250/**
6251 * Saves the auto load/store'd guest MSRs from the current VMCS into
6252 * the guest-CPU context.
6253 *
6254 * @returns VBox status code.
6255 * @param pVCpu Pointer to the VMCPU.
6256 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6257 * out-of-sync. Make sure to update the required fields
6258 * before using them.
6259 *
6260 * @remarks No-long-jump zone!!!
6261 */
6262static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6263{
6264 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6265 return VINF_SUCCESS;
6266
6267 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6268 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6269 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6270 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6271 {
6272 switch (pMsr->u32Msr)
6273 {
6274 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6275 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6276 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6277 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6278 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6279 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6280 break;
6281
6282 default:
6283 {
6284 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6285 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6286 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6287 }
6288 }
6289 }
6290
6291 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6292 return VINF_SUCCESS;
6293}
6294
6295
6296/**
6297 * Saves the guest control registers from the current VMCS into the guest-CPU
6298 * context.
6299 *
6300 * @returns VBox status code.
6301 * @param pVCpu Pointer to the VMCPU.
6302 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6303 * out-of-sync. Make sure to update the required fields
6304 * before using them.
6305 *
6306 * @remarks No-long-jump zone!!!
6307 */
6308static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6309{
6310 /* Guest CR0. Guest FPU. */
6311 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6312 AssertRCReturn(rc, rc);
6313
6314 /* Guest CR4. */
6315 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6316 AssertRCReturn(rc, rc);
6317
6318 /* Guest CR2 - updated always during the world-switch or in #PF. */
6319 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6320 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6321 {
6322 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6323 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6324
6325 PVM pVM = pVCpu->CTX_SUFF(pVM);
6326 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6327 || ( pVM->hm.s.fNestedPaging
6328 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6329 {
6330 uint64_t u64Val = 0;
6331 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6332 if (pMixedCtx->cr3 != u64Val)
6333 {
6334 CPUMSetGuestCR3(pVCpu, u64Val);
6335 if (VMMRZCallRing3IsEnabled(pVCpu))
6336 {
6337 PGMUpdateCR3(pVCpu, u64Val);
6338 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6339 }
6340 else
6341 {
6342 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6343 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6344 }
6345 }
6346
6347 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6348 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6349 {
6350 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6351 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6352 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6353 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6354
6355 if (VMMRZCallRing3IsEnabled(pVCpu))
6356 {
6357 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6358 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6359 }
6360 else
6361 {
6362 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6363 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6364 }
6365 }
6366 }
6367
6368 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6369 }
6370
6371 /*
6372 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6373 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6374 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6375 *
6376 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6377 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6378 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6379 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6380 *
6381 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6382 */
6383 if (VMMRZCallRing3IsEnabled(pVCpu))
6384 {
6385 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6386 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6387
6388 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6389 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6390
6391 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6392 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6393 }
6394
6395 return rc;
6396}
6397
6398
6399/**
6400 * Reads a guest segment register from the current VMCS into the guest-CPU
6401 * context.
6402 *
6403 * @returns VBox status code.
6404 * @param pVCpu Pointer to the VMCPU.
6405 * @param idxSel Index of the selector in the VMCS.
6406 * @param idxLimit Index of the segment limit in the VMCS.
6407 * @param idxBase Index of the segment base in the VMCS.
6408 * @param idxAccess Index of the access rights of the segment in the VMCS.
6409 * @param pSelReg Pointer to the segment selector.
6410 *
6411 * @remarks No-long-jump zone!!!
6412 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6413 * macro as that takes care of whether to read from the VMCS cache or
6414 * not.
6415 */
6416DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6417 PCPUMSELREG pSelReg)
6418{
6419 NOREF(pVCpu);
6420
6421 uint32_t u32Val = 0;
6422 int rc = VMXReadVmcs32(idxSel, &u32Val);
6423 AssertRCReturn(rc, rc);
6424 pSelReg->Sel = (uint16_t)u32Val;
6425 pSelReg->ValidSel = (uint16_t)u32Val;
6426 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6427
6428 rc = VMXReadVmcs32(idxLimit, &u32Val);
6429 AssertRCReturn(rc, rc);
6430 pSelReg->u32Limit = u32Val;
6431
6432 uint64_t u64Val = 0;
6433 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6434 AssertRCReturn(rc, rc);
6435 pSelReg->u64Base = u64Val;
6436
6437 rc = VMXReadVmcs32(idxAccess, &u32Val);
6438 AssertRCReturn(rc, rc);
6439 pSelReg->Attr.u = u32Val;
6440
6441 /*
6442 * If VT-x marks the segment as unusable, most other bits remain undefined:
6443 * - For CS the L, D and G bits have meaning.
6444 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6445 * - For the remaining data segments no bits are defined.
6446 *
6447 * The present bit and the unusable bit has been observed to be set at the
6448 * same time (the selector was supposed to be invalid as we started executing
6449 * a V8086 interrupt in ring-0).
6450 *
6451 * What should be important for the rest of the VBox code, is that the P bit is
6452 * cleared. Some of the other VBox code recognizes the unusable bit, but
6453 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6454 * safe side here, we'll strip off P and other bits we don't care about. If
6455 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6456 *
6457 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6458 */
6459 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6460 {
6461 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6462
6463 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6464 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6465 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6466
6467 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6468#ifdef DEBUG_bird
6469 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6470 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6471 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6472#endif
6473 }
6474 return VINF_SUCCESS;
6475}
6476
6477
6478#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6479# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6480 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6481 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6482#else
6483# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6484 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6485 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6486#endif
6487
6488
6489/**
6490 * Saves the guest segment registers from the current VMCS into the guest-CPU
6491 * context.
6492 *
6493 * @returns VBox status code.
6494 * @param pVCpu Pointer to the VMCPU.
6495 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6496 * out-of-sync. Make sure to update the required fields
6497 * before using them.
6498 *
6499 * @remarks No-long-jump zone!!!
6500 */
6501static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6502{
6503 /* Guest segment registers. */
6504 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6505 {
6506 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6507 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6508 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6509 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6510 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6511 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6512 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6513
6514 /* Restore segment attributes for real-on-v86 mode hack. */
6515 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6516 {
6517 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6518 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6519 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6520 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6521 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6522 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6523 }
6524 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6525 }
6526
6527 return VINF_SUCCESS;
6528}
6529
6530
6531/**
6532 * Saves the guest descriptor table registers and task register from the current
6533 * VMCS into the guest-CPU context.
6534 *
6535 * @returns VBox status code.
6536 * @param pVCpu Pointer to the VMCPU.
6537 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6538 * out-of-sync. Make sure to update the required fields
6539 * before using them.
6540 *
6541 * @remarks No-long-jump zone!!!
6542 */
6543static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6544{
6545 int rc = VINF_SUCCESS;
6546
6547 /* Guest LDTR. */
6548 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6549 {
6550 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6551 AssertRCReturn(rc, rc);
6552 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6553 }
6554
6555 /* Guest GDTR. */
6556 uint64_t u64Val = 0;
6557 uint32_t u32Val = 0;
6558 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6559 {
6560 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6561 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6562 pMixedCtx->gdtr.pGdt = u64Val;
6563 pMixedCtx->gdtr.cbGdt = u32Val;
6564 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6565 }
6566
6567 /* Guest IDTR. */
6568 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6569 {
6570 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6571 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6572 pMixedCtx->idtr.pIdt = u64Val;
6573 pMixedCtx->idtr.cbIdt = u32Val;
6574 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6575 }
6576
6577 /* Guest TR. */
6578 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6579 {
6580 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6581 AssertRCReturn(rc, rc);
6582
6583 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6584 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6585 {
6586 rc = VMXLOCAL_READ_SEG(TR, tr);
6587 AssertRCReturn(rc, rc);
6588 }
6589 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6590 }
6591 return rc;
6592}
6593
6594#undef VMXLOCAL_READ_SEG
6595
6596
6597/**
6598 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6599 * context.
6600 *
6601 * @returns VBox status code.
6602 * @param pVCpu Pointer to the VMCPU.
6603 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6604 * out-of-sync. Make sure to update the required fields
6605 * before using them.
6606 *
6607 * @remarks No-long-jump zone!!!
6608 */
6609static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6610{
6611 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6612 {
6613 if (!pVCpu->hm.s.fUsingHyperDR7)
6614 {
6615 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6616 uint32_t u32Val;
6617 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6618 pMixedCtx->dr[7] = u32Val;
6619 }
6620
6621 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6622 }
6623 return VINF_SUCCESS;
6624}
6625
6626
6627/**
6628 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6629 *
6630 * @returns VBox status code.
6631 * @param pVCpu Pointer to the VMCPU.
6632 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6633 * out-of-sync. Make sure to update the required fields
6634 * before using them.
6635 *
6636 * @remarks No-long-jump zone!!!
6637 */
6638static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6639{
6640 NOREF(pMixedCtx);
6641
6642 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6643 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6644 return VINF_SUCCESS;
6645}
6646
6647
6648/**
6649 * Saves the entire guest state from the currently active VMCS into the
6650 * guest-CPU context. This essentially VMREADs all guest-data.
6651 *
6652 * @returns VBox status code.
6653 * @param pVCpu Pointer to the VMCPU.
6654 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6655 * out-of-sync. Make sure to update the required fields
6656 * before using them.
6657 */
6658static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6659{
6660 Assert(pVCpu);
6661 Assert(pMixedCtx);
6662
6663 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6664 return VINF_SUCCESS;
6665
6666 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6667 again on the ring-3 callback path, there is no real need to. */
6668 if (VMMRZCallRing3IsEnabled(pVCpu))
6669 VMMR0LogFlushDisable(pVCpu);
6670 else
6671 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6672 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6673
6674 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6675 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6676
6677 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6678 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6679
6680 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6681 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6682
6683 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6684 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6685
6686 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6687 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6688
6689 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6690 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6691
6692 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6693 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6694
6695 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6696 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6697
6698 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6699 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6700
6701 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6702 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6703
6704 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6705 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6706
6707 if (VMMRZCallRing3IsEnabled(pVCpu))
6708 VMMR0LogFlushEnable(pVCpu);
6709
6710 return rc;
6711}
6712
6713
6714/**
6715 * Check per-VM and per-VCPU force flag actions that require us to go back to
6716 * ring-3 for one reason or another.
6717 *
6718 * @returns VBox status code (information status code included).
6719 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6720 * ring-3.
6721 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6722 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6723 * interrupts)
6724 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6725 * all EMTs to be in ring-3.
6726 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6727 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6728 * to the EM loop.
6729 *
6730 * @param pVM Pointer to the VM.
6731 * @param pVCpu Pointer to the VMCPU.
6732 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6733 * out-of-sync. Make sure to update the required fields
6734 * before using them.
6735 */
6736static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6737{
6738 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6739
6740 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6741 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6742 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6743 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6744 {
6745 /* We need the control registers now, make sure the guest-CPU context is updated. */
6746 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6747 AssertRCReturn(rc3, rc3);
6748
6749 /* Pending HM CR3 sync. */
6750 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6751 {
6752 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6753 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6754 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6755 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6756 }
6757
6758 /* Pending HM PAE PDPEs. */
6759 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6760 {
6761 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6762 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6763 }
6764
6765 /* Pending PGM C3 sync. */
6766 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6767 {
6768 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6769 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6770 if (rc2 != VINF_SUCCESS)
6771 {
6772 AssertRC(rc2);
6773 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6774 return rc2;
6775 }
6776 }
6777
6778 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6779 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6780 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6781 {
6782 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6783 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6784 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6785 return rc2;
6786 }
6787
6788 /* Pending VM request packets, such as hardware interrupts. */
6789 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6790 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6791 {
6792 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6793 return VINF_EM_PENDING_REQUEST;
6794 }
6795
6796 /* Pending PGM pool flushes. */
6797 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6798 {
6799 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6800 return VINF_PGM_POOL_FLUSH_PENDING;
6801 }
6802
6803 /* Pending DMA requests. */
6804 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6805 {
6806 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6807 return VINF_EM_RAW_TO_R3;
6808 }
6809 }
6810
6811 return VINF_SUCCESS;
6812}
6813
6814
6815/**
6816 * Converts any TRPM trap into a pending HM event. This is typically used when
6817 * entering from ring-3 (not longjmp returns).
6818 *
6819 * @param pVCpu Pointer to the VMCPU.
6820 */
6821static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6822{
6823 Assert(TRPMHasTrap(pVCpu));
6824 Assert(!pVCpu->hm.s.Event.fPending);
6825
6826 uint8_t uVector;
6827 TRPMEVENT enmTrpmEvent;
6828 RTGCUINT uErrCode;
6829 RTGCUINTPTR GCPtrFaultAddress;
6830 uint8_t cbInstr;
6831
6832 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6833 AssertRC(rc);
6834
6835 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6836 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6837 if (enmTrpmEvent == TRPM_TRAP)
6838 {
6839 switch (uVector)
6840 {
6841 case X86_XCPT_NMI:
6842 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6843 break;
6844
6845 case X86_XCPT_BP:
6846 case X86_XCPT_OF:
6847 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6848 break;
6849
6850 case X86_XCPT_PF:
6851 case X86_XCPT_DF:
6852 case X86_XCPT_TS:
6853 case X86_XCPT_NP:
6854 case X86_XCPT_SS:
6855 case X86_XCPT_GP:
6856 case X86_XCPT_AC:
6857 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6858 /* no break! */
6859 default:
6860 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6861 break;
6862 }
6863 }
6864 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6865 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6866 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6867 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6868 else
6869 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6870
6871 rc = TRPMResetTrap(pVCpu);
6872 AssertRC(rc);
6873 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6874 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6875
6876 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6877 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6878}
6879
6880
6881/**
6882 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6883 * VT-x to execute any instruction.
6884 *
6885 * @param pvCpu Pointer to the VMCPU.
6886 */
6887static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6888{
6889 Assert(pVCpu->hm.s.Event.fPending);
6890
6891 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6892 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6893 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6894 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6895
6896 /* If a trap was already pending, we did something wrong! */
6897 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6898
6899 TRPMEVENT enmTrapType;
6900 switch (uVectorType)
6901 {
6902 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6903 enmTrapType = TRPM_HARDWARE_INT;
6904 break;
6905
6906 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6907 enmTrapType = TRPM_SOFTWARE_INT;
6908 break;
6909
6910 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6911 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6912 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6913 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6914 enmTrapType = TRPM_TRAP;
6915 break;
6916
6917 default:
6918 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6919 enmTrapType = TRPM_32BIT_HACK;
6920 break;
6921 }
6922
6923 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6924
6925 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6926 AssertRC(rc);
6927
6928 if (fErrorCodeValid)
6929 TRPMSetErrorCode(pVCpu, uErrorCode);
6930
6931 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6932 && uVector == X86_XCPT_PF)
6933 {
6934 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6935 }
6936 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6937 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6938 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6939 {
6940 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6941 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6942 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6943 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6944 }
6945 pVCpu->hm.s.Event.fPending = false;
6946}
6947
6948
6949/**
6950 * Does the necessary state syncing before returning to ring-3 for any reason
6951 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6952 *
6953 * @returns VBox status code.
6954 * @param pVM Pointer to the VM.
6955 * @param pVCpu Pointer to the VMCPU.
6956 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6957 * be out-of-sync. Make sure to update the required
6958 * fields before using them.
6959 * @param fSaveGuestState Whether to save the guest state or not.
6960 *
6961 * @remarks No-long-jmp zone!!!
6962 */
6963static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6964{
6965 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6966 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6967
6968 RTCPUID idCpu = RTMpCpuId();
6969 Log4Func(("HostCpuId=%u\n", idCpu));
6970
6971 /*
6972 * !!! IMPORTANT !!!
6973 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6974 */
6975
6976 /* Save the guest state if necessary. */
6977 if ( fSaveGuestState
6978 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6979 {
6980 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6981 AssertRCReturn(rc, rc);
6982 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6983 }
6984
6985 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6986 if (CPUMIsGuestFPUStateActive(pVCpu))
6987 {
6988 /* We shouldn't reload CR0 without saving it first. */
6989 if (!fSaveGuestState)
6990 {
6991 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6992 AssertRCReturn(rc, rc);
6993 }
6994 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6995 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6996 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6997 }
6998
6999 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7000#ifdef VBOX_STRICT
7001 if (CPUMIsHyperDebugStateActive(pVCpu))
7002 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7003#endif
7004 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7005 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7006 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7007 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7008
7009#if HC_ARCH_BITS == 64
7010 /* Restore host-state bits that VT-x only restores partially. */
7011 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7012 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7013 {
7014 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7015 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7016 }
7017 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7018#endif
7019
7020#if HC_ARCH_BITS == 64
7021 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7022 if ( pVM->hm.s.fAllow64BitGuests
7023 && pVCpu->hm.s.vmx.fLazyMsrs)
7024 {
7025 /* We shouldn't reload the guest MSRs without saving it first. */
7026 if (!fSaveGuestState)
7027 {
7028 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7029 AssertRCReturn(rc, rc);
7030 }
7031 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7032 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7033 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7034 }
7035#endif
7036
7037 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7038 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7039
7040 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7041 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7042 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7043 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7044 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7045 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7046 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7047 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7048
7049 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7050
7051 /** @todo This partially defeats the purpose of having preemption hooks.
7052 * The problem is, deregistering the hooks should be moved to a place that
7053 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7054 * context.
7055 */
7056 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7057 {
7058 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7059 AssertRCReturn(rc, rc);
7060
7061 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7062 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7063 }
7064 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7065 NOREF(idCpu);
7066
7067 return VINF_SUCCESS;
7068}
7069
7070
7071/**
7072 * Leaves the VT-x session.
7073 *
7074 * @returns VBox status code.
7075 * @param pVM Pointer to the VM.
7076 * @param pVCpu Pointer to the VMCPU.
7077 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7078 * out-of-sync. Make sure to update the required fields
7079 * before using them.
7080 *
7081 * @remarks No-long-jmp zone!!!
7082 */
7083DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7084{
7085 HM_DISABLE_PREEMPT_IF_NEEDED();
7086 HMVMX_ASSERT_CPU_SAFE();
7087 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7088 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7089
7090 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7091 and done this from the VMXR0ThreadCtxCallback(). */
7092 if (!pVCpu->hm.s.fLeaveDone)
7093 {
7094 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7095 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7096 pVCpu->hm.s.fLeaveDone = true;
7097 }
7098 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7099
7100 /*
7101 * !!! IMPORTANT !!!
7102 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7103 */
7104
7105 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7106 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7107 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7108 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7109 VMMR0ThreadCtxHooksDeregister(pVCpu);
7110
7111 /* Leave HM context. This takes care of local init (term). */
7112 int rc = HMR0LeaveCpu(pVCpu);
7113
7114 HM_RESTORE_PREEMPT_IF_NEEDED();
7115
7116 return rc;
7117}
7118
7119
7120/**
7121 * Does the necessary state syncing before doing a longjmp to ring-3.
7122 *
7123 * @returns VBox status code.
7124 * @param pVM Pointer to the VM.
7125 * @param pVCpu Pointer to the VMCPU.
7126 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7127 * out-of-sync. Make sure to update the required fields
7128 * before using them.
7129 *
7130 * @remarks No-long-jmp zone!!!
7131 */
7132DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7133{
7134 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7135}
7136
7137
7138/**
7139 * Take necessary actions before going back to ring-3.
7140 *
7141 * An action requires us to go back to ring-3. This function does the necessary
7142 * steps before we can safely return to ring-3. This is not the same as longjmps
7143 * to ring-3, this is voluntary and prepares the guest so it may continue
7144 * executing outside HM (recompiler/IEM).
7145 *
7146 * @returns VBox status code.
7147 * @param pVM Pointer to the VM.
7148 * @param pVCpu Pointer to the VMCPU.
7149 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7150 * out-of-sync. Make sure to update the required fields
7151 * before using them.
7152 * @param rcExit The reason for exiting to ring-3. Can be
7153 * VINF_VMM_UNKNOWN_RING3_CALL.
7154 */
7155static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7156{
7157 Assert(pVM);
7158 Assert(pVCpu);
7159 Assert(pMixedCtx);
7160 HMVMX_ASSERT_PREEMPT_SAFE();
7161
7162 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7163 {
7164 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7165 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7166 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7167 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7168 }
7169
7170 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7171 VMMRZCallRing3Disable(pVCpu);
7172 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7173
7174 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7175 if (pVCpu->hm.s.Event.fPending)
7176 {
7177 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7178 Assert(!pVCpu->hm.s.Event.fPending);
7179 }
7180
7181 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7182 and if we're injecting an event we should have a TRPM trap pending. */
7183 Assert(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu));
7184 Assert(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu));
7185
7186 /* Save guest state and restore host state bits. */
7187 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7188 AssertRCReturn(rc, rc);
7189 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7190 /* Thread-context hooks are unregistered at this point!!! */
7191
7192 /* Sync recompiler state. */
7193 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7194 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7195 | CPUM_CHANGED_LDTR
7196 | CPUM_CHANGED_GDTR
7197 | CPUM_CHANGED_IDTR
7198 | CPUM_CHANGED_TR
7199 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7200 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7201 if ( pVM->hm.s.fNestedPaging
7202 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7203 {
7204 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7205 }
7206
7207 Assert(!pVCpu->hm.s.fClearTrapFlag);
7208
7209 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7210 if (rcExit != VINF_EM_RAW_INTERRUPT)
7211 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7212
7213 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7214
7215 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7216 VMMRZCallRing3RemoveNotification(pVCpu);
7217 VMMRZCallRing3Enable(pVCpu);
7218
7219 return rc;
7220}
7221
7222
7223/**
7224 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7225 * longjump to ring-3 and possibly get preempted.
7226 *
7227 * @returns VBox status code.
7228 * @param pVCpu Pointer to the VMCPU.
7229 * @param enmOperation The operation causing the ring-3 longjump.
7230 * @param pvUser Opaque pointer to the guest-CPU context. The data
7231 * may be out-of-sync. Make sure to update the required
7232 * fields before using them.
7233 */
7234DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7235{
7236 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7237 {
7238 /*
7239 * !!! IMPORTANT !!!
7240 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7241 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7242 */
7243 VMMRZCallRing3RemoveNotification(pVCpu);
7244 VMMRZCallRing3Disable(pVCpu);
7245 HM_DISABLE_PREEMPT_IF_NEEDED();
7246
7247 PVM pVM = pVCpu->CTX_SUFF(pVM);
7248 if (CPUMIsGuestFPUStateActive(pVCpu))
7249 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7250
7251 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7252
7253#if HC_ARCH_BITS == 64
7254 /* Restore host-state bits that VT-x only restores partially. */
7255 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7256 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7257 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7258 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7259
7260 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7261 if ( pVM->hm.s.fAllow64BitGuests
7262 && pVCpu->hm.s.vmx.fLazyMsrs)
7263 {
7264 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7265 }
7266#endif
7267 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7268 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7269 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7270 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7271 {
7272 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7273 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7274 }
7275
7276 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7277 VMMR0ThreadCtxHooksDeregister(pVCpu);
7278
7279 HMR0LeaveCpu(pVCpu);
7280 HM_RESTORE_PREEMPT_IF_NEEDED();
7281 return VINF_SUCCESS;
7282 }
7283
7284 Assert(pVCpu);
7285 Assert(pvUser);
7286 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7287 HMVMX_ASSERT_PREEMPT_SAFE();
7288
7289 VMMRZCallRing3Disable(pVCpu);
7290 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7291
7292 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7293 enmOperation));
7294
7295 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7296 AssertRCReturn(rc, rc);
7297
7298 VMMRZCallRing3Enable(pVCpu);
7299 return VINF_SUCCESS;
7300}
7301
7302
7303/**
7304 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7305 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7306 *
7307 * @param pVCpu Pointer to the VMCPU.
7308 */
7309DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7310{
7311 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7312 {
7313 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7314 {
7315 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7316 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7317 AssertRC(rc);
7318 Log4(("Setup interrupt-window exiting\n"));
7319 }
7320 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7321}
7322
7323
7324/**
7325 * Clears the interrupt-window exiting control in the VMCS.
7326 *
7327 * @param pVCpu Pointer to the VMCPU.
7328 */
7329DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7330{
7331 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7332 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7333 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7334 AssertRC(rc);
7335 Log4(("Cleared interrupt-window exiting\n"));
7336}
7337
7338
7339/**
7340 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7341 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7342 *
7343 * @param pVCpu Pointer to the VMCPU.
7344 */
7345DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7346{
7347 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7348 {
7349 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7350 {
7351 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7352 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7353 AssertRC(rc);
7354 Log4(("Setup NMI-window exiting\n"));
7355 }
7356 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7357}
7358
7359
7360/**
7361 * Clears the NMI-window exiting control in the VMCS.
7362 *
7363 * @param pVCpu Pointer to the VMCPU.
7364 */
7365DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7366{
7367 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7368 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7369 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7370 AssertRC(rc);
7371 Log4(("Cleared NMI-window exiting\n"));
7372}
7373
7374
7375/**
7376 * Evaluates the event to be delivered to the guest and sets it as the pending
7377 * event.
7378 *
7379 * @param pVCpu Pointer to the VMCPU.
7380 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7381 * out-of-sync. Make sure to update the required fields
7382 * before using them.
7383 */
7384static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7385{
7386 Assert(!pVCpu->hm.s.Event.fPending);
7387
7388 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7389 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7390 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7391 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7392 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7393
7394 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7395 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7396 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7397 Assert(!TRPMHasTrap(pVCpu));
7398
7399 /*
7400 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7401 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7402 */
7403 /** @todo SMI. SMIs take priority over NMIs. */
7404 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7405 {
7406 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7407 if ( !fBlockNmi
7408 && !fBlockSti
7409 && !fBlockMovSS)
7410 {
7411 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7412 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7413 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7414
7415 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7416 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7417 }
7418 else
7419 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7420 }
7421 /*
7422 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7423 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7424 */
7425 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7426 && !pVCpu->hm.s.fSingleInstruction)
7427 {
7428 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7429 AssertRC(rc);
7430 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7431 if ( !fBlockInt
7432 && !fBlockSti
7433 && !fBlockMovSS)
7434 {
7435 uint8_t u8Interrupt;
7436 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7437 if (RT_SUCCESS(rc))
7438 {
7439 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7440 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7441 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7442
7443 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7444 }
7445 else
7446 {
7447 /** @todo Does this actually happen? If not turn it into an assertion. */
7448 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7449 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7450 }
7451 }
7452 else
7453 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7454 }
7455}
7456
7457
7458/**
7459 * Sets a pending-debug exception to be delivered to the guest if the guest is
7460 * single-stepping.
7461 *
7462 * @param pVCpu Pointer to the VMCPU.
7463 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7464 * out-of-sync. Make sure to update the required fields
7465 * before using them.
7466 */
7467DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7468{
7469 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7470 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7471 {
7472 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7473 AssertRC(rc);
7474 }
7475}
7476
7477
7478/**
7479 * Injects any pending events into the guest if the guest is in a state to
7480 * receive them.
7481 *
7482 * @returns VBox status code (informational status codes included).
7483 * @param pVCpu Pointer to the VMCPU.
7484 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7485 * out-of-sync. Make sure to update the required fields
7486 * before using them.
7487 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7488 * return VINF_EM_DBG_STEPPED if the event was
7489 * dispatched directly.
7490 */
7491static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7492{
7493 HMVMX_ASSERT_PREEMPT_SAFE();
7494 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7495
7496 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7497 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7498 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7499 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7500
7501 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7502 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7503 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7504 Assert(!TRPMHasTrap(pVCpu));
7505
7506 int rc = VINF_SUCCESS;
7507 if (pVCpu->hm.s.Event.fPending)
7508 {
7509 /*
7510 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7511 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7512 * ended up enabling interrupts outside VT-x.
7513 */
7514 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7515 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7516 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7517 {
7518 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7519 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7520 }
7521
7522#ifdef VBOX_STRICT
7523 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7524 {
7525 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7526 Assert(!fBlockInt);
7527 Assert(!fBlockSti);
7528 Assert(!fBlockMovSS);
7529 }
7530 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7531 {
7532 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7533 Assert(!fBlockSti);
7534 Assert(!fBlockMovSS);
7535 Assert(!fBlockNmi);
7536 }
7537#endif
7538 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7539 (uint8_t)uIntType));
7540 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7541 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7542 AssertRCReturn(rc, rc);
7543
7544 /* Update the interruptibility-state as it could have been changed by
7545 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7546 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7547 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7548
7549#ifdef VBOX_WITH_STATISTICS
7550 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7551 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7552 else
7553 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7554#endif
7555 }
7556
7557 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7558 if ( fBlockSti
7559 || fBlockMovSS)
7560 {
7561 if ( !pVCpu->hm.s.fSingleInstruction
7562 && !DBGFIsStepping(pVCpu))
7563 {
7564 /*
7565 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7566 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7567 * See Intel spec. 27.3.4 "Saving Non-Register State".
7568 */
7569 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7570 AssertRCReturn(rc2, rc2);
7571 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7572 }
7573 else if (pMixedCtx->eflags.Bits.u1TF)
7574 {
7575 /*
7576 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7577 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7578 */
7579 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7580 uIntrState = 0;
7581 }
7582 }
7583
7584 /*
7585 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7586 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7587 */
7588 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7589 AssertRC(rc2);
7590
7591 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7592 NOREF(fBlockMovSS); NOREF(fBlockSti);
7593 return rc;
7594}
7595
7596
7597/**
7598 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7599 *
7600 * @param pVCpu Pointer to the VMCPU.
7601 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7602 * out-of-sync. Make sure to update the required fields
7603 * before using them.
7604 */
7605DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7606{
7607 NOREF(pMixedCtx);
7608 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7609 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7610}
7611
7612
7613/**
7614 * Injects a double-fault (#DF) exception into the VM.
7615 *
7616 * @returns VBox status code (informational status code included).
7617 * @param pVCpu Pointer to the VMCPU.
7618 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7619 * out-of-sync. Make sure to update the required fields
7620 * before using them.
7621 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7622 * and should return VINF_EM_DBG_STEPPED if the event
7623 * is injected directly (register modified by us, not
7624 * by hardware on VM-entry).
7625 * @param puIntrState Pointer to the current guest interruptibility-state.
7626 * This interruptibility-state will be updated if
7627 * necessary. This cannot not be NULL.
7628 */
7629DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7630{
7631 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7632 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7633 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7634 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7635 fStepping, puIntrState);
7636}
7637
7638
7639/**
7640 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7641 *
7642 * @param pVCpu Pointer to the VMCPU.
7643 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7644 * out-of-sync. Make sure to update the required fields
7645 * before using them.
7646 */
7647DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7648{
7649 NOREF(pMixedCtx);
7650 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7651 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7652 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7653}
7654
7655
7656/**
7657 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7658 *
7659 * @param pVCpu Pointer to the VMCPU.
7660 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7661 * out-of-sync. Make sure to update the required fields
7662 * before using them.
7663 * @param cbInstr The value of RIP that is to be pushed on the guest
7664 * stack.
7665 */
7666DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7667{
7668 NOREF(pMixedCtx);
7669 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7670 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7671 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7672}
7673
7674
7675/**
7676 * Injects a general-protection (#GP) fault into the VM.
7677 *
7678 * @returns VBox status code (informational status code included).
7679 * @param pVCpu Pointer to the VMCPU.
7680 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7681 * out-of-sync. Make sure to update the required fields
7682 * before using them.
7683 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7684 * mode, i.e. in real-mode it's not valid).
7685 * @param u32ErrorCode The error code associated with the #GP.
7686 * @param fStepping Whether we're running in
7687 * hmR0VmxRunGuestCodeStep() and should return
7688 * VINF_EM_DBG_STEPPED if the event is injected
7689 * directly (register modified by us, not by
7690 * hardware on VM-entry).
7691 * @param puIntrState Pointer to the current guest interruptibility-state.
7692 * This interruptibility-state will be updated if
7693 * necessary. This cannot not be NULL.
7694 */
7695DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7696 bool fStepping, uint32_t *puIntrState)
7697{
7698 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7699 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7700 if (fErrorCodeValid)
7701 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7702 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7703 fStepping, puIntrState);
7704}
7705
7706
7707/**
7708 * Sets a general-protection (#GP) exception as pending-for-injection into the
7709 * VM.
7710 *
7711 * @param pVCpu Pointer to the VMCPU.
7712 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7713 * out-of-sync. Make sure to update the required fields
7714 * before using them.
7715 * @param u32ErrorCode The error code associated with the #GP.
7716 */
7717DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7718{
7719 NOREF(pMixedCtx);
7720 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7721 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7722 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7723 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7724}
7725
7726
7727/**
7728 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7729 *
7730 * @param pVCpu Pointer to the VMCPU.
7731 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7732 * out-of-sync. Make sure to update the required fields
7733 * before using them.
7734 * @param uVector The software interrupt vector number.
7735 * @param cbInstr The value of RIP that is to be pushed on the guest
7736 * stack.
7737 */
7738DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7739{
7740 NOREF(pMixedCtx);
7741 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7742 if ( uVector == X86_XCPT_BP
7743 || uVector == X86_XCPT_OF)
7744 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7745 else
7746 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7747 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7748}
7749
7750
7751/**
7752 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7753 * stack.
7754 *
7755 * @returns VBox status code (information status code included).
7756 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7757 * @param pVM Pointer to the VM.
7758 * @param pMixedCtx Pointer to the guest-CPU context.
7759 * @param uValue The value to push to the guest stack.
7760 */
7761DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7762{
7763 /*
7764 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7765 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7766 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7767 */
7768 if (pMixedCtx->sp == 1)
7769 return VINF_EM_RESET;
7770 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7771 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7772 AssertRCReturn(rc, rc);
7773 return rc;
7774}
7775
7776
7777/**
7778 * Injects an event into the guest upon VM-entry by updating the relevant fields
7779 * in the VM-entry area in the VMCS.
7780 *
7781 * @returns VBox status code (informational error codes included).
7782 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7783 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7784 *
7785 * @param pVCpu Pointer to the VMCPU.
7786 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7787 * be out-of-sync. Make sure to update the required
7788 * fields before using them.
7789 * @param u64IntInfo The VM-entry interruption-information field.
7790 * @param cbInstr The VM-entry instruction length in bytes (for
7791 * software interrupts, exceptions and privileged
7792 * software exceptions).
7793 * @param u32ErrCode The VM-entry exception error code.
7794 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7795 * @param puIntrState Pointer to the current guest interruptibility-state.
7796 * This interruptibility-state will be updated if
7797 * necessary. This cannot not be NULL.
7798 * @param fStepping Whether we're running in
7799 * hmR0VmxRunGuestCodeStep() and should return
7800 * VINF_EM_DBG_STEPPED if the event is injected
7801 * directly (register modified by us, not by
7802 * hardware on VM-entry).
7803 *
7804 * @remarks Requires CR0!
7805 * @remarks No-long-jump zone!!!
7806 */
7807static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7808 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7809{
7810 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7811 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7812 Assert(puIntrState);
7813 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7814
7815 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7816 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7817
7818#ifdef VBOX_STRICT
7819 /* Validate the error-code-valid bit for hardware exceptions. */
7820 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7821 {
7822 switch (uVector)
7823 {
7824 case X86_XCPT_PF:
7825 case X86_XCPT_DF:
7826 case X86_XCPT_TS:
7827 case X86_XCPT_NP:
7828 case X86_XCPT_SS:
7829 case X86_XCPT_GP:
7830 case X86_XCPT_AC:
7831 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7832 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7833 /* fallthru */
7834 default:
7835 break;
7836 }
7837 }
7838#endif
7839
7840 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7841 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7842 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7843
7844 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7845
7846 /* We require CR0 to check if the guest is in real-mode. */
7847 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7848 AssertRCReturn(rc, rc);
7849
7850 /*
7851 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7852 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7853 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7854 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7855 */
7856 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7857 {
7858 PVM pVM = pVCpu->CTX_SUFF(pVM);
7859 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7860 {
7861 Assert(PDMVmmDevHeapIsEnabled(pVM));
7862 Assert(pVM->hm.s.vmx.pRealModeTSS);
7863
7864 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7865 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7866 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7867 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7868 AssertRCReturn(rc, rc);
7869 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7870
7871 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7872 size_t const cbIdtEntry = sizeof(X86IDTR16);
7873 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7874 {
7875 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7876 if (uVector == X86_XCPT_DF)
7877 return VINF_EM_RESET;
7878
7879 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7880 if (uVector == X86_XCPT_GP)
7881 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7882
7883 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7884 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7885 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7886 fStepping, puIntrState);
7887 }
7888
7889 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7890 uint16_t uGuestIp = pMixedCtx->ip;
7891 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7892 {
7893 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7894 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7895 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7896 }
7897 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7898 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7899
7900 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7901 X86IDTR16 IdtEntry;
7902 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7903 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7904 AssertRCReturn(rc, rc);
7905
7906 /* Construct the stack frame for the interrupt/exception handler. */
7907 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7908 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7909 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7910 AssertRCReturn(rc, rc);
7911
7912 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7913 if (rc == VINF_SUCCESS)
7914 {
7915 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7916 pMixedCtx->rip = IdtEntry.offSel;
7917 pMixedCtx->cs.Sel = IdtEntry.uSel;
7918 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7919 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7920 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7921 && uVector == X86_XCPT_PF)
7922 pMixedCtx->cr2 = GCPtrFaultAddress;
7923
7924 /* If any other guest-state bits are changed here, make sure to update
7925 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7926 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7927 | HM_CHANGED_GUEST_RIP
7928 | HM_CHANGED_GUEST_RFLAGS
7929 | HM_CHANGED_GUEST_RSP);
7930
7931 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7932 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7933 {
7934 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7935 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7936 Log4(("Clearing inhibition due to STI.\n"));
7937 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7938 }
7939 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7940 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7941
7942 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7943 it, if we are returning to ring-3 before executing guest code. */
7944 pVCpu->hm.s.Event.fPending = false;
7945
7946 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7947 if (fStepping)
7948 rc = VINF_EM_DBG_STEPPED;
7949 }
7950 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7951 return rc;
7952 }
7953
7954 /*
7955 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7956 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7957 */
7958 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7959 }
7960
7961 /* Validate. */
7962 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7963 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7964 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7965
7966 /* Inject. */
7967 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7968 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7969 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7970 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7971
7972 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7973 && uVector == X86_XCPT_PF)
7974 pMixedCtx->cr2 = GCPtrFaultAddress;
7975
7976 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7977 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7978
7979 AssertRCReturn(rc, rc);
7980 return rc;
7981}
7982
7983
7984/**
7985 * Clears the interrupt-window exiting control in the VMCS and if necessary
7986 * clears the current event in the VMCS as well.
7987 *
7988 * @returns VBox status code.
7989 * @param pVCpu Pointer to the VMCPU.
7990 *
7991 * @remarks Use this function only to clear events that have not yet been
7992 * delivered to the guest but are injected in the VMCS!
7993 * @remarks No-long-jump zone!!!
7994 */
7995static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7996{
7997 int rc;
7998 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7999
8000 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8001 {
8002 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8003 Assert(!pVCpu->hm.s.Event.fPending);
8004 }
8005
8006 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8007 {
8008 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8009 Assert(!pVCpu->hm.s.Event.fPending);
8010 }
8011
8012 if (!pVCpu->hm.s.Event.fPending)
8013 return;
8014
8015#ifdef VBOX_STRICT
8016 uint32_t u32EntryInfo;
8017 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8018 AssertRC(rc);
8019 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8020#endif
8021
8022 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8023 AssertRC(rc);
8024
8025 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8026 AssertRC(rc);
8027
8028 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8029 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8030}
8031
8032
8033/**
8034 * Enters the VT-x session.
8035 *
8036 * @returns VBox status code.
8037 * @param pVM Pointer to the VM.
8038 * @param pVCpu Pointer to the VMCPU.
8039 * @param pCpu Pointer to the CPU info struct.
8040 */
8041VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8042{
8043 AssertPtr(pVM);
8044 AssertPtr(pVCpu);
8045 Assert(pVM->hm.s.vmx.fSupported);
8046 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8047 NOREF(pCpu); NOREF(pVM);
8048
8049 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8050 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8051
8052#ifdef VBOX_STRICT
8053 /* Make sure we're in VMX root mode. */
8054 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8055 if (!(u32HostCR4 & X86_CR4_VMXE))
8056 {
8057 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8058 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8059 }
8060#endif
8061
8062 /*
8063 * Load the VCPU's VMCS as the current (and active) one.
8064 */
8065 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8066 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8067 if (RT_FAILURE(rc))
8068 return rc;
8069
8070 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8071 pVCpu->hm.s.fLeaveDone = false;
8072 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8073
8074 return VINF_SUCCESS;
8075}
8076
8077
8078/**
8079 * The thread-context callback (only on platforms which support it).
8080 *
8081 * @param enmEvent The thread-context event.
8082 * @param pVCpu Pointer to the VMCPU.
8083 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8084 * @thread EMT(pVCpu)
8085 */
8086VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8087{
8088 NOREF(fGlobalInit);
8089
8090 switch (enmEvent)
8091 {
8092 case RTTHREADCTXEVENT_PREEMPTING:
8093 {
8094 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8095 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8096 VMCPU_ASSERT_EMT(pVCpu);
8097
8098 PVM pVM = pVCpu->CTX_SUFF(pVM);
8099 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8100
8101 /* No longjmps (logger flushes, locks) in this fragile context. */
8102 VMMRZCallRing3Disable(pVCpu);
8103 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8104
8105 /*
8106 * Restore host-state (FPU, debug etc.)
8107 */
8108 if (!pVCpu->hm.s.fLeaveDone)
8109 {
8110 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8111 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8112 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8113 pVCpu->hm.s.fLeaveDone = true;
8114 }
8115
8116 /* Leave HM context, takes care of local init (term). */
8117 int rc = HMR0LeaveCpu(pVCpu);
8118 AssertRC(rc); NOREF(rc);
8119
8120 /* Restore longjmp state. */
8121 VMMRZCallRing3Enable(pVCpu);
8122 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8123 break;
8124 }
8125
8126 case RTTHREADCTXEVENT_RESUMED:
8127 {
8128 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8129 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8130 VMCPU_ASSERT_EMT(pVCpu);
8131
8132 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8133 VMMRZCallRing3Disable(pVCpu);
8134 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8135
8136 /* Initialize the bare minimum state required for HM. This takes care of
8137 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8138 int rc = HMR0EnterCpu(pVCpu);
8139 AssertRC(rc);
8140 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8141
8142 /* Load the active VMCS as the current one. */
8143 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8144 {
8145 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8146 AssertRC(rc); NOREF(rc);
8147 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8148 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8149 }
8150 pVCpu->hm.s.fLeaveDone = false;
8151
8152 /* Restore longjmp state. */
8153 VMMRZCallRing3Enable(pVCpu);
8154 break;
8155 }
8156
8157 default:
8158 break;
8159 }
8160}
8161
8162
8163/**
8164 * Saves the host state in the VMCS host-state.
8165 * Sets up the VM-exit MSR-load area.
8166 *
8167 * The CPU state will be loaded from these fields on every successful VM-exit.
8168 *
8169 * @returns VBox status code.
8170 * @param pVM Pointer to the VM.
8171 * @param pVCpu Pointer to the VMCPU.
8172 *
8173 * @remarks No-long-jump zone!!!
8174 */
8175static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8176{
8177 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8178
8179 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8180 return VINF_SUCCESS;
8181
8182 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8183 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8184
8185 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8186 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8187
8188 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8189 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8190
8191 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8192 return rc;
8193}
8194
8195
8196/**
8197 * Saves the host state in the VMCS host-state.
8198 *
8199 * @returns VBox status code.
8200 * @param pVM Pointer to the VM.
8201 * @param pVCpu Pointer to the VMCPU.
8202 *
8203 * @remarks No-long-jump zone!!!
8204 */
8205VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8206{
8207 AssertPtr(pVM);
8208 AssertPtr(pVCpu);
8209
8210 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8211
8212 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8213 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8214 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8215 return hmR0VmxSaveHostState(pVM, pVCpu);
8216}
8217
8218
8219/**
8220 * Loads the guest state into the VMCS guest-state area.
8221 *
8222 * The will typically be done before VM-entry when the guest-CPU state and the
8223 * VMCS state may potentially be out of sync.
8224 *
8225 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8226 * VM-entry controls.
8227 * Sets up the appropriate VMX non-root function to execute guest code based on
8228 * the guest CPU mode.
8229 *
8230 * @returns VBox status code.
8231 * @param pVM Pointer to the VM.
8232 * @param pVCpu Pointer to the VMCPU.
8233 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8234 * out-of-sync. Make sure to update the required fields
8235 * before using them.
8236 *
8237 * @remarks No-long-jump zone!!!
8238 */
8239static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8240{
8241 AssertPtr(pVM);
8242 AssertPtr(pVCpu);
8243 AssertPtr(pMixedCtx);
8244 HMVMX_ASSERT_PREEMPT_SAFE();
8245
8246 VMMRZCallRing3Disable(pVCpu);
8247 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8248
8249 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8250
8251 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8252
8253 /* Determine real-on-v86 mode. */
8254 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8255 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8256 && CPUMIsGuestInRealModeEx(pMixedCtx))
8257 {
8258 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8259 }
8260
8261 /*
8262 * Load the guest-state into the VMCS.
8263 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8264 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8265 */
8266 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8267 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8268
8269 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8270 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8271 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8272
8273 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8274 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8275 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8276
8277 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8278 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8279
8280 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8281 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8282
8283 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8284 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8285 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8286
8287 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8288 determine we don't have to swap EFER after all. */
8289 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8290 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8291
8292 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8293 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8294
8295 /*
8296 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8297 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8298 */
8299 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8300 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8301
8302 /* Clear any unused and reserved bits. */
8303 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8304
8305 VMMRZCallRing3Enable(pVCpu);
8306
8307 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8308 return rc;
8309}
8310
8311
8312/**
8313 * Loads the state shared between the host and guest into the VMCS.
8314 *
8315 * @param pVM Pointer to the VM.
8316 * @param pVCpu Pointer to the VMCPU.
8317 * @param pCtx Pointer to the guest-CPU context.
8318 *
8319 * @remarks No-long-jump zone!!!
8320 */
8321static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8322{
8323 NOREF(pVM);
8324
8325 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8326 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8327
8328 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8329 {
8330 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8331 AssertRC(rc);
8332 }
8333
8334 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8335 {
8336 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8337 AssertRC(rc);
8338
8339 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8340 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8341 {
8342 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8343 AssertRC(rc);
8344 }
8345 }
8346
8347 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8348 {
8349#if HC_ARCH_BITS == 64
8350 if (pVM->hm.s.fAllow64BitGuests)
8351 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8352#endif
8353 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8354 }
8355
8356 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8357 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8358}
8359
8360
8361/**
8362 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8363 *
8364 * @param pVM Pointer to the VM.
8365 * @param pVCpu Pointer to the VMCPU.
8366 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8367 * out-of-sync. Make sure to update the required fields
8368 * before using them.
8369 */
8370DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8371{
8372 HMVMX_ASSERT_PREEMPT_SAFE();
8373
8374 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8375#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8376 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8377#endif
8378
8379 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8380 {
8381 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8382 AssertRC(rc);
8383 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8384 }
8385 else if (HMCPU_CF_VALUE(pVCpu))
8386 {
8387 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8388 AssertRC(rc);
8389 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8390 }
8391
8392 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8393 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8394 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8395 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8396}
8397
8398
8399/**
8400 * Does the preparations before executing guest code in VT-x.
8401 *
8402 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8403 * recompiler/IEM. We must be cautious what we do here regarding committing
8404 * guest-state information into the VMCS assuming we assuredly execute the
8405 * guest in VT-x mode.
8406 *
8407 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8408 * the common-state (TRPM/forceflags), we must undo those changes so that the
8409 * recompiler/IEM can (and should) use them when it resumes guest execution.
8410 * Otherwise such operations must be done when we can no longer exit to ring-3.
8411 *
8412 * @returns Strict VBox status code.
8413 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8414 * have been disabled.
8415 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8416 * double-fault into the guest.
8417 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8418 * dispatched directly.
8419 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8420 *
8421 * @param pVM Pointer to the VM.
8422 * @param pVCpu Pointer to the VMCPU.
8423 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8424 * out-of-sync. Make sure to update the required fields
8425 * before using them.
8426 * @param pVmxTransient Pointer to the VMX transient structure.
8427 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8428 * us ignore some of the reasons for returning to
8429 * ring-3, and return VINF_EM_DBG_STEPPED if event
8430 * dispatching took place.
8431 */
8432static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8433{
8434 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8435
8436#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8437 PGMRZDynMapFlushAutoSet(pVCpu);
8438#endif
8439
8440 /* Check force flag actions that might require us to go back to ring-3. */
8441 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8442 if (rc != VINF_SUCCESS)
8443 return rc;
8444
8445#ifndef IEM_VERIFICATION_MODE_FULL
8446 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8447 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8448 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8449 {
8450 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8451 RTGCPHYS GCPhysApicBase;
8452 GCPhysApicBase = pMixedCtx->msrApicBase;
8453 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8454
8455 /* Unalias any existing mapping. */
8456 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8457 AssertRCReturn(rc, rc);
8458
8459 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8460 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8461 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8462 AssertRCReturn(rc, rc);
8463
8464 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8465 }
8466#endif /* !IEM_VERIFICATION_MODE_FULL */
8467
8468 if (TRPMHasTrap(pVCpu))
8469 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8470 else if (!pVCpu->hm.s.Event.fPending)
8471 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8472
8473 /*
8474 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8475 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8476 */
8477 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8478 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8479 {
8480 Assert(rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
8481 return rc;
8482 }
8483
8484 /*
8485 * Load the guest state bits, we can handle longjmps/getting preempted here.
8486 *
8487 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8488 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8489 * Hence, this needs to be done -after- injection of events.
8490 */
8491 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8492
8493 /*
8494 * No longjmps to ring-3 from this point on!!!
8495 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8496 * This also disables flushing of the R0-logger instance (if any).
8497 */
8498 VMMRZCallRing3Disable(pVCpu);
8499
8500 /*
8501 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8502 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8503 *
8504 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8505 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8506 *
8507 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8508 * executing guest code.
8509 */
8510 pVmxTransient->uEflags = ASMIntDisableFlags();
8511 if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8512 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8513 && ( !fStepping /* Optimized for the non-stepping case, of course. */
8514 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8515 {
8516 hmR0VmxClearEventVmcs(pVCpu);
8517 ASMSetFlags(pVmxTransient->uEflags);
8518 VMMRZCallRing3Enable(pVCpu);
8519 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8520 return VINF_EM_RAW_TO_R3;
8521 }
8522
8523 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8524 {
8525 hmR0VmxClearEventVmcs(pVCpu);
8526 ASMSetFlags(pVmxTransient->uEflags);
8527 VMMRZCallRing3Enable(pVCpu);
8528 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8529 return VINF_EM_RAW_INTERRUPT;
8530 }
8531
8532 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8533 pVCpu->hm.s.Event.fPending = false;
8534
8535 return VINF_SUCCESS;
8536}
8537
8538
8539/**
8540 * Prepares to run guest code in VT-x and we've committed to doing so. This
8541 * means there is no backing out to ring-3 or anywhere else at this
8542 * point.
8543 *
8544 * @param pVM Pointer to the VM.
8545 * @param pVCpu Pointer to the VMCPU.
8546 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8547 * out-of-sync. Make sure to update the required fields
8548 * before using them.
8549 * @param pVmxTransient Pointer to the VMX transient structure.
8550 *
8551 * @remarks Called with preemption disabled.
8552 * @remarks No-long-jump zone!!!
8553 */
8554static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8555{
8556 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8557 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8558 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8559
8560 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8561 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8562
8563#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8564 if (!CPUMIsGuestFPUStateActive(pVCpu))
8565 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8566 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8567#endif
8568
8569 if ( pVCpu->hm.s.fUseGuestFpu
8570 && !CPUMIsGuestFPUStateActive(pVCpu))
8571 {
8572 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8573 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8574 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8575 }
8576
8577 /*
8578 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8579 */
8580 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8581 && pVCpu->hm.s.vmx.cMsrs > 0)
8582 {
8583 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8584 }
8585
8586 /*
8587 * Load the host state bits as we may've been preempted (only happens when
8588 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8589 */
8590 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8591 {
8592 /* This ASSUMES that pfnStartVM has been set up already. */
8593 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8594 AssertRC(rc);
8595 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8596 }
8597 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8598
8599 /*
8600 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8601 */
8602 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8603 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8604 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8605
8606 /* Store status of the shared guest-host state at the time of VM-entry. */
8607#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8608 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8609 {
8610 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8611 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8612 }
8613 else
8614#endif
8615 {
8616 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8617 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8618 }
8619 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8620
8621 /*
8622 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8623 */
8624 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8625 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8626
8627 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8628 RTCPUID idCurrentCpu = pCpu->idCpu;
8629 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8630 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8631 {
8632 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8633 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8634 }
8635
8636 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8637 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8638 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8639 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8640
8641 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8642
8643 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8644 to start executing. */
8645
8646 /*
8647 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8648 */
8649 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8650 {
8651 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8652 {
8653 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8654 AssertRC(rc2);
8655 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8656 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8657 true /* fUpdateHostMsr */);
8658 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8659 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8660 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8661 }
8662 else
8663 {
8664 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8665 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8666 }
8667 }
8668
8669#ifdef VBOX_STRICT
8670 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8671 hmR0VmxCheckHostEferMsr(pVCpu);
8672 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8673#endif
8674#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8675 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8676 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8677 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8678#endif
8679}
8680
8681
8682/**
8683 * Performs some essential restoration of state after running guest code in
8684 * VT-x.
8685 *
8686 * @param pVM Pointer to the VM.
8687 * @param pVCpu Pointer to the VMCPU.
8688 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8689 * out-of-sync. Make sure to update the required fields
8690 * before using them.
8691 * @param pVmxTransient Pointer to the VMX transient structure.
8692 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8693 *
8694 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8695 *
8696 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8697 * unconditionally when it is safe to do so.
8698 */
8699static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8700{
8701 NOREF(pVM);
8702
8703 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8704
8705 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8706 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8707 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8708 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8709 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8710 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8711
8712 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8713 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset); /** @todo use SUPReadTSC() eventually. */
8714
8715 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8716 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8717 Assert(!(ASMGetFlags() & X86_EFL_IF));
8718 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8719
8720#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8721 if (CPUMIsGuestFPUStateActive(pVCpu))
8722 {
8723 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8724 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8725 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8726 }
8727#endif
8728
8729#if HC_ARCH_BITS == 64
8730 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8731#endif
8732 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8733#ifdef VBOX_STRICT
8734 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8735#endif
8736 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8737 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8738
8739 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8740 uint32_t uExitReason;
8741 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8742 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8743 AssertRC(rc);
8744 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8745 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8746
8747 /* Update the VM-exit history array. */
8748 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8749
8750 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8751 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8752 {
8753 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8754 pVmxTransient->fVMEntryFailed));
8755 return;
8756 }
8757
8758 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8759 {
8760 /** @todo We can optimize this by only syncing with our force-flags when
8761 * really needed and keeping the VMCS state as it is for most
8762 * VM-exits. */
8763 /* Update the guest interruptibility-state from the VMCS. */
8764 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8765
8766#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8767 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8768 AssertRC(rc);
8769#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8770 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8771 AssertRC(rc);
8772#endif
8773
8774 /*
8775 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8776 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8777 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8778 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8779 */
8780 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8781 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8782 {
8783 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8784 AssertRC(rc);
8785 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8786 }
8787 }
8788}
8789
8790
8791/**
8792 * Runs the guest code using VT-x the normal way.
8793 *
8794 * @returns VBox status code.
8795 * @param pVM Pointer to the VM.
8796 * @param pVCpu Pointer to the VMCPU.
8797 * @param pCtx Pointer to the guest-CPU context.
8798 *
8799 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8800 */
8801static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8802{
8803 VMXTRANSIENT VmxTransient;
8804 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8805 int rc = VERR_INTERNAL_ERROR_5;
8806 uint32_t cLoops = 0;
8807
8808 for (;; cLoops++)
8809 {
8810 Assert(!HMR0SuspendPending());
8811 HMVMX_ASSERT_CPU_SAFE();
8812
8813 /* Preparatory work for running guest code, this may force us to return
8814 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8815 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8816 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8817 if (rc != VINF_SUCCESS)
8818 break;
8819
8820 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8821 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8822 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8823
8824 /* Restore any residual host-state and save any bits shared between host
8825 and guest into the guest-CPU state. Re-enables interrupts! */
8826 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8827
8828 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8829 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8830 {
8831 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8832 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8833 return rc;
8834 }
8835
8836 /* Profile the VM-exit. */
8837 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8838 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8839 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8840 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8841 HMVMX_START_EXIT_DISPATCH_PROF();
8842
8843 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8844 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8845 {
8846 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8847 hmR0VmxSaveGuestState(pVCpu, pCtx);
8848 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8849 }
8850
8851 /* Handle the VM-exit. */
8852#ifdef HMVMX_USE_FUNCTION_TABLE
8853 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8854#else
8855 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8856#endif
8857 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8858 if (rc != VINF_SUCCESS)
8859 break;
8860 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8861 {
8862 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8863 rc = VINF_EM_RAW_INTERRUPT;
8864 break;
8865 }
8866 }
8867
8868 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8869 return rc;
8870}
8871
8872
8873/**
8874 * Single steps guest code using VT-x.
8875 *
8876 * @returns VBox status code.
8877 * @param pVM Pointer to the VM.
8878 * @param pVCpu Pointer to the VMCPU.
8879 * @param pCtx Pointer to the guest-CPU context.
8880 *
8881 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8882 */
8883static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8884{
8885 VMXTRANSIENT VmxTransient;
8886 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8887 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8888 uint32_t cLoops = 0;
8889 uint16_t uCsStart = pCtx->cs.Sel;
8890 uint64_t uRipStart = pCtx->rip;
8891
8892 for (;; cLoops++)
8893 {
8894 Assert(!HMR0SuspendPending());
8895 HMVMX_ASSERT_CPU_SAFE();
8896
8897 /* Preparatory work for running guest code, this may force us to return
8898 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8899 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8900 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8901 if (rcStrict != VINF_SUCCESS)
8902 break;
8903
8904 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8905 rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8906 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8907
8908 /* Restore any residual host-state and save any bits shared between host
8909 and guest into the guest-CPU state. Re-enables interrupts! */
8910 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8911
8912 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8913 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
8914 {
8915 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8916 hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
8917 return VBOXSTRICTRC_TODO(rcStrict);
8918 }
8919
8920 /* Profile the VM-exit. */
8921 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8922 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8923 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8924 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8925 HMVMX_START_EXIT_DISPATCH_PROF();
8926
8927 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8928 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8929 {
8930 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8931 hmR0VmxSaveGuestState(pVCpu, pCtx);
8932 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8933 }
8934
8935 /* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8936 rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8937 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8938 if (rcStrict != VINF_SUCCESS)
8939 break;
8940 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8941 {
8942 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8943 rcStrict = VINF_EM_RAW_INTERRUPT;
8944 break;
8945 }
8946
8947 /*
8948 * Did the RIP change, if so, consider it a single step.
8949 * Otherwise, make sure one of the TFs gets set.
8950 */
8951 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8952 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8953 AssertRCReturn(rc2, rc2);
8954 if ( pCtx->rip != uRipStart
8955 || pCtx->cs.Sel != uCsStart)
8956 {
8957 rcStrict = VINF_EM_DBG_STEPPED;
8958 break;
8959 }
8960 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8961 }
8962
8963 /*
8964 * Clear the X86_EFL_TF if necessary.
8965 */
8966 if (pVCpu->hm.s.fClearTrapFlag)
8967 {
8968 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8969 AssertRCReturn(rc2, rc2);
8970 pVCpu->hm.s.fClearTrapFlag = false;
8971 pCtx->eflags.Bits.u1TF = 0;
8972 }
8973 /** @todo there seems to be issues with the resume flag when the monitor trap
8974 * flag is pending without being used. Seen early in bios init when
8975 * accessing APIC page in protected mode. */
8976
8977 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8978 return VBOXSTRICTRC_TODO(rcStrict);
8979}
8980
8981
8982/**
8983 * Runs the guest code using VT-x.
8984 *
8985 * @returns VBox status code.
8986 * @param pVM Pointer to the VM.
8987 * @param pVCpu Pointer to the VMCPU.
8988 * @param pCtx Pointer to the guest-CPU context.
8989 */
8990VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8991{
8992 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8993 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8994 HMVMX_ASSERT_PREEMPT_SAFE();
8995
8996 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8997
8998 int rc;
8999 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
9000 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
9001 else
9002 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
9003
9004 if (rc == VERR_EM_INTERPRETER)
9005 rc = VINF_EM_RAW_EMULATE_INSTR;
9006 else if (rc == VINF_EM_RESET)
9007 rc = VINF_EM_TRIPLE_FAULT;
9008
9009 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
9010 if (RT_FAILURE(rc2))
9011 {
9012 pVCpu->hm.s.u32HMError = rc;
9013 rc = rc2;
9014 }
9015 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
9016 return rc;
9017}
9018
9019
9020#ifndef HMVMX_USE_FUNCTION_TABLE
9021DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9022{
9023#ifdef DEBUG_ramshankar
9024# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9025# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9026#endif
9027 int rc;
9028 switch (rcReason)
9029 {
9030 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9031 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9032 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9033 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9034 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9035 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9036 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9037 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9038 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9039 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9040 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9041 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9042 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9043 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9044 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9045 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9046 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9047 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9048 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9049 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9050 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9051 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9052 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9053 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9054 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9055 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9056 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9057 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9058 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9059 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9060 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9061 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9062 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9063 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9064
9065 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9066 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9067 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9068 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9069 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9070 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9071 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9072 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9073 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9074
9075 case VMX_EXIT_VMCLEAR:
9076 case VMX_EXIT_VMLAUNCH:
9077 case VMX_EXIT_VMPTRLD:
9078 case VMX_EXIT_VMPTRST:
9079 case VMX_EXIT_VMREAD:
9080 case VMX_EXIT_VMRESUME:
9081 case VMX_EXIT_VMWRITE:
9082 case VMX_EXIT_VMXOFF:
9083 case VMX_EXIT_VMXON:
9084 case VMX_EXIT_INVEPT:
9085 case VMX_EXIT_INVVPID:
9086 case VMX_EXIT_VMFUNC:
9087 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9088 break;
9089 default:
9090 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9091 break;
9092 }
9093 return rc;
9094}
9095#endif /* !HMVMX_USE_FUNCTION_TABLE */
9096
9097
9098/**
9099 * Single-stepping VM-exit filtering.
9100 *
9101 * This is preprocessing the exits and deciding whether we've gotten far enough
9102 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9103 * performed.
9104 *
9105 * @returns Strict VBox status code.
9106 * @param pVCpu The virtual CPU of the calling EMT.
9107 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9108 * out-of-sync. Make sure to update the required
9109 * fields before using them.
9110 * @param pVmxTransient Pointer to the VMX-transient structure.
9111 * @param uExitReason The VM-exit reason.
9112 */
9113DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9114 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9115{
9116 switch (uExitReason)
9117 {
9118 case VMX_EXIT_XCPT_OR_NMI:
9119 {
9120 /* Check for host NMI. */
9121 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9122 AssertRCReturn(rc2, rc2);
9123 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9124 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9125 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9126 /* fall thru */
9127 }
9128
9129 case VMX_EXIT_EPT_MISCONFIG:
9130 case VMX_EXIT_TRIPLE_FAULT:
9131 case VMX_EXIT_APIC_ACCESS:
9132 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9133 case VMX_EXIT_TASK_SWITCH:
9134
9135 /* Instruction specific VM-exits: */
9136 case VMX_EXIT_IO_INSTR:
9137 case VMX_EXIT_CPUID:
9138 case VMX_EXIT_RDTSC:
9139 case VMX_EXIT_RDTSCP:
9140 case VMX_EXIT_MOV_CRX:
9141 case VMX_EXIT_MWAIT:
9142 case VMX_EXIT_MONITOR:
9143 case VMX_EXIT_RDMSR:
9144 case VMX_EXIT_WRMSR:
9145 case VMX_EXIT_MOV_DRX:
9146 case VMX_EXIT_HLT:
9147 case VMX_EXIT_INVD:
9148 case VMX_EXIT_INVLPG:
9149 case VMX_EXIT_RSM:
9150 case VMX_EXIT_PAUSE:
9151 case VMX_EXIT_XDTR_ACCESS:
9152 case VMX_EXIT_TR_ACCESS:
9153 case VMX_EXIT_WBINVD:
9154 case VMX_EXIT_XSETBV:
9155 case VMX_EXIT_RDRAND:
9156 case VMX_EXIT_INVPCID:
9157 case VMX_EXIT_GETSEC:
9158 case VMX_EXIT_RDPMC:
9159 case VMX_EXIT_VMCALL:
9160 case VMX_EXIT_VMCLEAR:
9161 case VMX_EXIT_VMLAUNCH:
9162 case VMX_EXIT_VMPTRLD:
9163 case VMX_EXIT_VMPTRST:
9164 case VMX_EXIT_VMREAD:
9165 case VMX_EXIT_VMRESUME:
9166 case VMX_EXIT_VMWRITE:
9167 case VMX_EXIT_VMXOFF:
9168 case VMX_EXIT_VMXON:
9169 case VMX_EXIT_INVEPT:
9170 case VMX_EXIT_INVVPID:
9171 case VMX_EXIT_VMFUNC:
9172 {
9173 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9174 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9175 AssertRCReturn(rc2, rc2);
9176 if ( pMixedCtx->rip != uRipStart
9177 || pMixedCtx->cs.Sel != uCsStart)
9178 return VINF_EM_DBG_STEPPED;
9179 break;
9180 }
9181 }
9182
9183 /*
9184 * Normal processing.
9185 */
9186#ifdef HMVMX_USE_FUNCTION_TABLE
9187 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9188#else
9189 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9190#endif
9191}
9192
9193
9194#ifdef DEBUG
9195/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9196# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9197 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9198
9199# define HMVMX_ASSERT_PREEMPT_CPUID() \
9200 do \
9201 { \
9202 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9203 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9204 } while (0)
9205
9206# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9207 do { \
9208 AssertPtr(pVCpu); \
9209 AssertPtr(pMixedCtx); \
9210 AssertPtr(pVmxTransient); \
9211 Assert(pVmxTransient->fVMEntryFailed == false); \
9212 Assert(ASMIntAreEnabled()); \
9213 HMVMX_ASSERT_PREEMPT_SAFE(); \
9214 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9215 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9216 HMVMX_ASSERT_PREEMPT_SAFE(); \
9217 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9218 HMVMX_ASSERT_PREEMPT_CPUID(); \
9219 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9220 } while (0)
9221
9222# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9223 do { \
9224 Log4Func(("\n")); \
9225 } while (0)
9226#else /* Release builds */
9227# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9228 do { \
9229 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9230 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9231 } while (0)
9232# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9233#endif
9234
9235
9236/**
9237 * Advances the guest RIP after reading it from the VMCS.
9238 *
9239 * @returns VBox status code.
9240 * @param pVCpu Pointer to the VMCPU.
9241 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9242 * out-of-sync. Make sure to update the required fields
9243 * before using them.
9244 * @param pVmxTransient Pointer to the VMX transient structure.
9245 *
9246 * @remarks No-long-jump zone!!!
9247 */
9248DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9249{
9250 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9251 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9252 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9253 AssertRCReturn(rc, rc);
9254
9255 pMixedCtx->rip += pVmxTransient->cbInstr;
9256 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9257
9258 /*
9259 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9260 * pending debug exception field as it takes care of priority of events.
9261 *
9262 * See Intel spec. 32.2.1 "Debug Exceptions".
9263 */
9264 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9265
9266 return rc;
9267}
9268
9269
9270/**
9271 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9272 * and update error record fields accordingly.
9273 *
9274 * @return VMX_IGS_* return codes.
9275 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9276 * wrong with the guest state.
9277 *
9278 * @param pVM Pointer to the VM.
9279 * @param pVCpu Pointer to the VMCPU.
9280 * @param pCtx Pointer to the guest-CPU state.
9281 *
9282 * @remarks This function assumes our cache of the VMCS controls
9283 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9284 */
9285static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9286{
9287#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9288#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9289 uError = (err); \
9290 break; \
9291 } else do { } while (0)
9292
9293 int rc;
9294 uint32_t uError = VMX_IGS_ERROR;
9295 uint32_t u32Val;
9296 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9297
9298 do
9299 {
9300 /*
9301 * CR0.
9302 */
9303 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9304 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9305 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9306 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9307 if (fUnrestrictedGuest)
9308 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9309
9310 uint32_t u32GuestCR0;
9311 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9312 AssertRCBreak(rc);
9313 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9314 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9315 if ( !fUnrestrictedGuest
9316 && (u32GuestCR0 & X86_CR0_PG)
9317 && !(u32GuestCR0 & X86_CR0_PE))
9318 {
9319 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9320 }
9321
9322 /*
9323 * CR4.
9324 */
9325 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9326 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9327
9328 uint32_t u32GuestCR4;
9329 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9330 AssertRCBreak(rc);
9331 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9332 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9333
9334 /*
9335 * IA32_DEBUGCTL MSR.
9336 */
9337 uint64_t u64Val;
9338 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9339 AssertRCBreak(rc);
9340 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9341 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9342 {
9343 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9344 }
9345 uint64_t u64DebugCtlMsr = u64Val;
9346
9347#ifdef VBOX_STRICT
9348 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9349 AssertRCBreak(rc);
9350 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9351#endif
9352 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9353
9354 /*
9355 * RIP and RFLAGS.
9356 */
9357 uint32_t u32Eflags;
9358#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9359 if (HMVMX_IS_64BIT_HOST_MODE())
9360 {
9361 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9362 AssertRCBreak(rc);
9363 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9364 if ( !fLongModeGuest
9365 || !pCtx->cs.Attr.n.u1Long)
9366 {
9367 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9368 }
9369 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9370 * must be identical if the "IA-32e mode guest" VM-entry
9371 * control is 1 and CS.L is 1. No check applies if the
9372 * CPU supports 64 linear-address bits. */
9373
9374 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9375 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9376 AssertRCBreak(rc);
9377 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9378 VMX_IGS_RFLAGS_RESERVED);
9379 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9380 u32Eflags = u64Val;
9381 }
9382 else
9383#endif
9384 {
9385 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9386 AssertRCBreak(rc);
9387 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9388 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9389 }
9390
9391 if ( fLongModeGuest
9392 || ( fUnrestrictedGuest
9393 && !(u32GuestCR0 & X86_CR0_PE)))
9394 {
9395 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9396 }
9397
9398 uint32_t u32EntryInfo;
9399 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9400 AssertRCBreak(rc);
9401 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9402 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9403 {
9404 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9405 }
9406
9407 /*
9408 * 64-bit checks.
9409 */
9410#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9411 if (HMVMX_IS_64BIT_HOST_MODE())
9412 {
9413 if ( fLongModeGuest
9414 && !fUnrestrictedGuest)
9415 {
9416 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9417 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9418 }
9419
9420 if ( !fLongModeGuest
9421 && (u32GuestCR4 & X86_CR4_PCIDE))
9422 {
9423 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9424 }
9425
9426 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9427 * 51:32 beyond the processor's physical-address width are 0. */
9428
9429 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9430 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9431 {
9432 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9433 }
9434
9435 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9436 AssertRCBreak(rc);
9437 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9438
9439 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9440 AssertRCBreak(rc);
9441 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9442 }
9443#endif
9444
9445 /*
9446 * PERF_GLOBAL MSR.
9447 */
9448 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9449 {
9450 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9451 AssertRCBreak(rc);
9452 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9453 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9454 }
9455
9456 /*
9457 * PAT MSR.
9458 */
9459 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9460 {
9461 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9462 AssertRCBreak(rc);
9463 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9464 for (unsigned i = 0; i < 8; i++)
9465 {
9466 uint8_t u8Val = (u64Val & 0xff);
9467 if ( u8Val != 0 /* UC */
9468 && u8Val != 1 /* WC */
9469 && u8Val != 4 /* WT */
9470 && u8Val != 5 /* WP */
9471 && u8Val != 6 /* WB */
9472 && u8Val != 7 /* UC- */)
9473 {
9474 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9475 }
9476 u64Val >>= 8;
9477 }
9478 }
9479
9480 /*
9481 * EFER MSR.
9482 */
9483 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9484 {
9485 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9486 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9487 AssertRCBreak(rc);
9488 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9489 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9490 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9491 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9492 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9493 || !(u32GuestCR0 & X86_CR0_PG)
9494 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9495 VMX_IGS_EFER_LMA_LME_MISMATCH);
9496 }
9497
9498 /*
9499 * Segment registers.
9500 */
9501 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9502 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9503 if (!(u32Eflags & X86_EFL_VM))
9504 {
9505 /* CS */
9506 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9507 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9508 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9509 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9510 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9511 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9512 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9513 /* CS cannot be loaded with NULL in protected mode. */
9514 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9515 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9516 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9517 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9518 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9519 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9520 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9521 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9522 else
9523 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9524
9525 /* SS */
9526 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9527 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9528 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9529 if ( !(pCtx->cr0 & X86_CR0_PE)
9530 || pCtx->cs.Attr.n.u4Type == 3)
9531 {
9532 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9533 }
9534 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9535 {
9536 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9537 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9538 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9539 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9540 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9541 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9542 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9543 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9544 }
9545
9546 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9547 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9548 {
9549 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9550 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9551 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9552 || pCtx->ds.Attr.n.u4Type > 11
9553 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9554 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9555 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9556 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9557 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9558 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9559 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9560 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9561 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9562 }
9563 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9564 {
9565 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9566 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9567 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9568 || pCtx->es.Attr.n.u4Type > 11
9569 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9570 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9571 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9572 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9573 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9574 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9575 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9576 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9577 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9578 }
9579 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9580 {
9581 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9582 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9583 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9584 || pCtx->fs.Attr.n.u4Type > 11
9585 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9586 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9587 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9588 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9589 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9590 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9591 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9592 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9593 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9594 }
9595 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9596 {
9597 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9598 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9599 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9600 || pCtx->gs.Attr.n.u4Type > 11
9601 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9602 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9603 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9604 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9605 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9606 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9607 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9608 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9609 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9610 }
9611 /* 64-bit capable CPUs. */
9612#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9613 if (HMVMX_IS_64BIT_HOST_MODE())
9614 {
9615 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9616 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9617 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9618 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9619 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9620 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9621 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9622 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9623 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9624 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9625 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9626 }
9627#endif
9628 }
9629 else
9630 {
9631 /* V86 mode checks. */
9632 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9633 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9634 {
9635 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9636 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9637 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9638 }
9639 else
9640 {
9641 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9642 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9643 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9644 }
9645
9646 /* CS */
9647 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9648 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9649 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9650 /* SS */
9651 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9652 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9653 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9654 /* DS */
9655 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9656 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9657 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9658 /* ES */
9659 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9660 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9661 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9662 /* FS */
9663 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9664 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9665 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9666 /* GS */
9667 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9668 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9669 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9670 /* 64-bit capable CPUs. */
9671#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9672 if (HMVMX_IS_64BIT_HOST_MODE())
9673 {
9674 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9675 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9676 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9677 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9678 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9679 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9680 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9681 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9682 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9683 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9684 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9685 }
9686#endif
9687 }
9688
9689 /*
9690 * TR.
9691 */
9692 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9693 /* 64-bit capable CPUs. */
9694#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9695 if (HMVMX_IS_64BIT_HOST_MODE())
9696 {
9697 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9698 }
9699#endif
9700 if (fLongModeGuest)
9701 {
9702 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9703 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9704 }
9705 else
9706 {
9707 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9708 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9709 VMX_IGS_TR_ATTR_TYPE_INVALID);
9710 }
9711 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9712 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9713 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9714 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9715 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9716 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9717 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9718 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9719
9720 /*
9721 * GDTR and IDTR.
9722 */
9723#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9724 if (HMVMX_IS_64BIT_HOST_MODE())
9725 {
9726 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9727 AssertRCBreak(rc);
9728 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9729
9730 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9731 AssertRCBreak(rc);
9732 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9733 }
9734#endif
9735
9736 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9737 AssertRCBreak(rc);
9738 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9739
9740 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9741 AssertRCBreak(rc);
9742 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9743
9744 /*
9745 * Guest Non-Register State.
9746 */
9747 /* Activity State. */
9748 uint32_t u32ActivityState;
9749 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9750 AssertRCBreak(rc);
9751 HMVMX_CHECK_BREAK( !u32ActivityState
9752 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9753 VMX_IGS_ACTIVITY_STATE_INVALID);
9754 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9755 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9756 uint32_t u32IntrState;
9757 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9758 AssertRCBreak(rc);
9759 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9760 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9761 {
9762 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9763 }
9764
9765 /** @todo Activity state and injecting interrupts. Left as a todo since we
9766 * currently don't use activity states but ACTIVE. */
9767
9768 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9769 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9770
9771 /* Guest interruptibility-state. */
9772 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9773 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9774 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9775 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9776 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9777 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9778 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9779 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9780 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9781 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9782 {
9783 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9784 {
9785 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9786 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9787 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9788 }
9789 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9790 {
9791 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9792 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9793 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9794 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9795 }
9796 }
9797 /** @todo Assumes the processor is not in SMM. */
9798 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9799 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9800 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9801 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9802 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9803 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9804 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9805 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9806 {
9807 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9808 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9809 }
9810
9811 /* Pending debug exceptions. */
9812 if (HMVMX_IS_64BIT_HOST_MODE())
9813 {
9814 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9815 AssertRCBreak(rc);
9816 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9817 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9818 u32Val = u64Val; /* For pending debug exceptions checks below. */
9819 }
9820 else
9821 {
9822 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9823 AssertRCBreak(rc);
9824 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9825 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9826 }
9827
9828 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9829 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9830 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9831 {
9832 if ( (u32Eflags & X86_EFL_TF)
9833 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9834 {
9835 /* Bit 14 is PendingDebug.BS. */
9836 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9837 }
9838 if ( !(u32Eflags & X86_EFL_TF)
9839 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9840 {
9841 /* Bit 14 is PendingDebug.BS. */
9842 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9843 }
9844 }
9845
9846 /* VMCS link pointer. */
9847 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9848 AssertRCBreak(rc);
9849 if (u64Val != UINT64_C(0xffffffffffffffff))
9850 {
9851 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9852 /** @todo Bits beyond the processor's physical-address width MBZ. */
9853 /** @todo 32-bit located in memory referenced by value of this field (as a
9854 * physical address) must contain the processor's VMCS revision ID. */
9855 /** @todo SMM checks. */
9856 }
9857
9858 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9859 * not using Nested Paging? */
9860 if ( pVM->hm.s.fNestedPaging
9861 && !fLongModeGuest
9862 && CPUMIsGuestInPAEModeEx(pCtx))
9863 {
9864 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9865 AssertRCBreak(rc);
9866 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9867
9868 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9869 AssertRCBreak(rc);
9870 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9871
9872 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9873 AssertRCBreak(rc);
9874 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9875
9876 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9877 AssertRCBreak(rc);
9878 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9879 }
9880
9881 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9882 if (uError == VMX_IGS_ERROR)
9883 uError = VMX_IGS_REASON_NOT_FOUND;
9884 } while (0);
9885
9886 pVCpu->hm.s.u32HMError = uError;
9887 return uError;
9888
9889#undef HMVMX_ERROR_BREAK
9890#undef HMVMX_CHECK_BREAK
9891}
9892
9893/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9894/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9895/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9896
9897/** @name VM-exit handlers.
9898 * @{
9899 */
9900
9901/**
9902 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9903 */
9904HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9905{
9906 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9907 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9908 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9909 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9910 return VINF_SUCCESS;
9911 return VINF_EM_RAW_INTERRUPT;
9912}
9913
9914
9915/**
9916 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9917 */
9918HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9919{
9920 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9921 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9922
9923 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9924 AssertRCReturn(rc, rc);
9925
9926 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9927 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9928 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9929 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9930
9931 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9932 {
9933 /*
9934 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9935 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9936 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9937 *
9938 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9939 */
9940 VMXDispatchHostNmi();
9941 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9942 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9943 return VINF_SUCCESS;
9944 }
9945
9946 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9947 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9948 if (RT_UNLIKELY(rc != VINF_SUCCESS))
9949 {
9950 if (rc == VINF_HM_DOUBLE_FAULT)
9951 rc = VINF_SUCCESS;
9952 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9953 return rc;
9954 }
9955
9956 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9957 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9958 switch (uIntType)
9959 {
9960 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9961 Assert(uVector == X86_XCPT_DB);
9962 /* no break */
9963 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9964 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9965 /* no break */
9966 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9967 {
9968 switch (uVector)
9969 {
9970 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9971 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9972 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9973 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9974 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9975 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9976#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9977 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9978 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9979 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9980 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9981 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9982 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9983 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9984 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9985 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9986 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9987 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9988 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9989#endif
9990 default:
9991 {
9992 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9993 AssertRCReturn(rc, rc);
9994
9995 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9996 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9997 {
9998 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9999 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10000 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10001
10002 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10003 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10004 AssertRCReturn(rc, rc);
10005 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10006 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10007 0 /* GCPtrFaultAddress */);
10008 AssertRCReturn(rc, rc);
10009 }
10010 else
10011 {
10012 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10013 pVCpu->hm.s.u32HMError = uVector;
10014 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10015 }
10016 break;
10017 }
10018 }
10019 break;
10020 }
10021
10022 default:
10023 {
10024 pVCpu->hm.s.u32HMError = uExitIntInfo;
10025 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10026 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10027 break;
10028 }
10029 }
10030 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10031 return rc;
10032}
10033
10034
10035/**
10036 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10037 */
10038HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10039{
10040 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10041
10042 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
10043 hmR0VmxClearIntWindowExitVmcs(pVCpu);
10044
10045 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10046 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
10047 return VINF_SUCCESS;
10048}
10049
10050
10051/**
10052 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
10053 */
10054HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10055{
10056 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10057 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
10058 {
10059 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
10060 HMVMX_RETURN_UNEXPECTED_EXIT();
10061 }
10062
10063 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
10064
10065 /*
10066 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
10067 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
10068 */
10069 uint32_t uIntrState = 0;
10070 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10071 AssertRCReturn(rc, rc);
10072
10073 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
10074 if ( fBlockSti
10075 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
10076 {
10077 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10078 }
10079
10080 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
10081 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
10082
10083 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10084 return VINF_SUCCESS;
10085}
10086
10087
10088/**
10089 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10090 */
10091HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10092{
10093 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10094 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10095 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10096}
10097
10098
10099/**
10100 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10101 */
10102HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10103{
10104 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10105 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10106 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10107}
10108
10109
10110/**
10111 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10112 */
10113HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10114{
10115 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10116 PVM pVM = pVCpu->CTX_SUFF(pVM);
10117 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10118 if (RT_LIKELY(rc == VINF_SUCCESS))
10119 {
10120 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10121 Assert(pVmxTransient->cbInstr == 2);
10122 }
10123 else
10124 {
10125 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10126 rc = VERR_EM_INTERPRETER;
10127 }
10128 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10129 return rc;
10130}
10131
10132
10133/**
10134 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10135 */
10136HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10137{
10138 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10139 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10140 AssertRCReturn(rc, rc);
10141
10142 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10143 return VINF_EM_RAW_EMULATE_INSTR;
10144
10145 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10146 HMVMX_RETURN_UNEXPECTED_EXIT();
10147}
10148
10149
10150/**
10151 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10152 */
10153HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10154{
10155 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10156 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10157 AssertRCReturn(rc, rc);
10158
10159 PVM pVM = pVCpu->CTX_SUFF(pVM);
10160 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10161 if (RT_LIKELY(rc == VINF_SUCCESS))
10162 {
10163 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10164 Assert(pVmxTransient->cbInstr == 2);
10165 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10166 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10167 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10168 }
10169 else
10170 rc = VERR_EM_INTERPRETER;
10171 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10172 return rc;
10173}
10174
10175
10176/**
10177 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10178 */
10179HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10180{
10181 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10182 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10183 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10184 AssertRCReturn(rc, rc);
10185
10186 PVM pVM = pVCpu->CTX_SUFF(pVM);
10187 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10188 if (RT_LIKELY(rc == VINF_SUCCESS))
10189 {
10190 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10191 Assert(pVmxTransient->cbInstr == 3);
10192 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10193 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10194 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10195 }
10196 else
10197 {
10198 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10199 rc = VERR_EM_INTERPRETER;
10200 }
10201 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10202 return rc;
10203}
10204
10205
10206/**
10207 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10208 */
10209HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10210{
10211 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10212 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10213 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10214 AssertRCReturn(rc, rc);
10215
10216 PVM pVM = pVCpu->CTX_SUFF(pVM);
10217 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10218 if (RT_LIKELY(rc == VINF_SUCCESS))
10219 {
10220 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10221 Assert(pVmxTransient->cbInstr == 2);
10222 }
10223 else
10224 {
10225 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10226 rc = VERR_EM_INTERPRETER;
10227 }
10228 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10229 return rc;
10230}
10231
10232
10233/**
10234 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10235 */
10236HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10237{
10238 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10239
10240 int rc = VERR_NOT_SUPPORTED;
10241 if (GIMAreHypercallsEnabled(pVCpu))
10242 {
10243 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10244 AssertRCReturn(rc, rc);
10245
10246 rc = GIMHypercall(pVCpu, pMixedCtx);
10247 }
10248 if (rc != VINF_SUCCESS)
10249 {
10250 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10251 rc = VINF_SUCCESS;
10252 }
10253
10254 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10255 return rc;
10256}
10257
10258
10259/**
10260 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10261 */
10262HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10263{
10264 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10265 PVM pVM = pVCpu->CTX_SUFF(pVM);
10266 Assert(!pVM->hm.s.fNestedPaging);
10267
10268 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10269 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10270 AssertRCReturn(rc, rc);
10271
10272 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10273 rc = VBOXSTRICTRC_VAL(rc2);
10274 if (RT_LIKELY(rc == VINF_SUCCESS))
10275 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10276 else
10277 {
10278 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10279 pVmxTransient->uExitQualification, rc));
10280 }
10281 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10282 return rc;
10283}
10284
10285
10286/**
10287 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10288 */
10289HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10290{
10291 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10292 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10293 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10294 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10295 AssertRCReturn(rc, rc);
10296
10297 PVM pVM = pVCpu->CTX_SUFF(pVM);
10298 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10299 if (RT_LIKELY(rc == VINF_SUCCESS))
10300 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10301 else
10302 {
10303 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10304 rc = VERR_EM_INTERPRETER;
10305 }
10306 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10307 return rc;
10308}
10309
10310
10311/**
10312 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10313 */
10314HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10315{
10316 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10317 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10318 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10319 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10320 AssertRCReturn(rc, rc);
10321
10322 PVM pVM = pVCpu->CTX_SUFF(pVM);
10323 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10324 rc = VBOXSTRICTRC_VAL(rc2);
10325 if (RT_LIKELY( rc == VINF_SUCCESS
10326 || rc == VINF_EM_HALT))
10327 {
10328 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10329 AssertRCReturn(rc3, rc3);
10330
10331 if ( rc == VINF_EM_HALT
10332 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10333 {
10334 rc = VINF_SUCCESS;
10335 }
10336 }
10337 else
10338 {
10339 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10340 rc = VERR_EM_INTERPRETER;
10341 }
10342 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10343 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10344 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10345 return rc;
10346}
10347
10348
10349/**
10350 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10351 */
10352HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10353{
10354 /*
10355 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10356 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10357 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10358 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10359 */
10360 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10361 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10362 HMVMX_RETURN_UNEXPECTED_EXIT();
10363}
10364
10365
10366/**
10367 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10368 */
10369HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10370{
10371 /*
10372 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10373 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10374 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10375 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10376 */
10377 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10378 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10379 HMVMX_RETURN_UNEXPECTED_EXIT();
10380}
10381
10382
10383/**
10384 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10385 */
10386HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10387{
10388 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10389 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10390 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10391 HMVMX_RETURN_UNEXPECTED_EXIT();
10392}
10393
10394
10395/**
10396 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10397 */
10398HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10399{
10400 /*
10401 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10402 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10403 * See Intel spec. 25.3 "Other Causes of VM-exits".
10404 */
10405 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10406 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10407 HMVMX_RETURN_UNEXPECTED_EXIT();
10408}
10409
10410
10411/**
10412 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10413 * VM-exit.
10414 */
10415HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10416{
10417 /*
10418 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10419 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10420 *
10421 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10422 * See Intel spec. "23.8 Restrictions on VMX operation".
10423 */
10424 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10425 return VINF_SUCCESS;
10426}
10427
10428
10429/**
10430 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10431 * VM-exit.
10432 */
10433HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10434{
10435 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10436 return VINF_EM_RESET;
10437}
10438
10439
10440/**
10441 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10442 */
10443HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10444{
10445 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10446 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10447 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10448 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10449 AssertRCReturn(rc, rc);
10450
10451 pMixedCtx->rip++;
10452 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10453 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10454 rc = VINF_SUCCESS;
10455 else
10456 rc = VINF_EM_HALT;
10457
10458 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10459 if (rc != VINF_SUCCESS)
10460 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHltToR3);
10461 return rc;
10462}
10463
10464
10465/**
10466 * VM-exit handler for instructions that result in a #UD exception delivered to
10467 * the guest.
10468 */
10469HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10470{
10471 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10472 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10473 return VINF_SUCCESS;
10474}
10475
10476
10477/**
10478 * VM-exit handler for expiry of the VMX preemption timer.
10479 */
10480HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10481{
10482 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10483
10484 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10485 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10486
10487 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10488 PVM pVM = pVCpu->CTX_SUFF(pVM);
10489 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10490 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10491 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10492}
10493
10494
10495/**
10496 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10497 */
10498HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10499{
10500 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10501
10502 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10503 /** @todo check if XSETBV is supported by the recompiler. */
10504 return VERR_EM_INTERPRETER;
10505}
10506
10507
10508/**
10509 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10510 */
10511HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10512{
10513 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10514
10515 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10516 /** @todo implement EMInterpretInvpcid() */
10517 return VERR_EM_INTERPRETER;
10518}
10519
10520
10521/**
10522 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10523 * Error VM-exit.
10524 */
10525HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10526{
10527 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10528 AssertRCReturn(rc, rc);
10529
10530 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10531 AssertRCReturn(rc, rc);
10532
10533 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10534 NOREF(uInvalidReason);
10535
10536#ifdef VBOX_STRICT
10537 uint32_t uIntrState;
10538 HMVMXHCUINTREG uHCReg;
10539 uint64_t u64Val;
10540 uint32_t u32Val;
10541
10542 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10543 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10544 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10545 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10546 AssertRCReturn(rc, rc);
10547
10548 Log4(("uInvalidReason %u\n", uInvalidReason));
10549 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10550 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10551 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10552 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10553
10554 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10555 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10556 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10557 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10558 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10559 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10560 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10561 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10562 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10563 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10564 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10565 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10566#else
10567 NOREF(pVmxTransient);
10568#endif
10569
10570 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10571 return VERR_VMX_INVALID_GUEST_STATE;
10572}
10573
10574
10575/**
10576 * VM-exit handler for VM-entry failure due to an MSR-load
10577 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10578 */
10579HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10580{
10581 NOREF(pVmxTransient);
10582 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10583 HMVMX_RETURN_UNEXPECTED_EXIT();
10584}
10585
10586
10587/**
10588 * VM-exit handler for VM-entry failure due to a machine-check event
10589 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10590 */
10591HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10592{
10593 NOREF(pVmxTransient);
10594 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10595 HMVMX_RETURN_UNEXPECTED_EXIT();
10596}
10597
10598
10599/**
10600 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10601 * theory.
10602 */
10603HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10604{
10605 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10606 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10607 return VERR_VMX_UNDEFINED_EXIT_CODE;
10608}
10609
10610
10611/**
10612 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10613 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10614 * Conditional VM-exit.
10615 */
10616HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10617{
10618 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10619
10620 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10621 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10622 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10623 return VERR_EM_INTERPRETER;
10624 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10625 HMVMX_RETURN_UNEXPECTED_EXIT();
10626}
10627
10628
10629/**
10630 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10631 */
10632HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10633{
10634 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10635
10636 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10637 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10638 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10639 return VERR_EM_INTERPRETER;
10640 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10641 HMVMX_RETURN_UNEXPECTED_EXIT();
10642}
10643
10644
10645/**
10646 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10647 */
10648HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10649{
10650 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10651
10652 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10653 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10654 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10655 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10656 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10657 {
10658 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10659 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10660 }
10661 AssertRCReturn(rc, rc);
10662 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10663
10664#ifdef VBOX_STRICT
10665 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10666 {
10667 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10668 && pMixedCtx->ecx != MSR_K6_EFER)
10669 {
10670 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10671 HMVMX_RETURN_UNEXPECTED_EXIT();
10672 }
10673# if HC_ARCH_BITS == 64
10674 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10675 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10676 {
10677 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10678 HMVMX_RETURN_UNEXPECTED_EXIT();
10679 }
10680# endif
10681 }
10682#endif
10683
10684 PVM pVM = pVCpu->CTX_SUFF(pVM);
10685 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10686 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10687 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10688 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10689 if (RT_LIKELY(rc == VINF_SUCCESS))
10690 {
10691 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10692 Assert(pVmxTransient->cbInstr == 2);
10693 }
10694 return rc;
10695}
10696
10697
10698/**
10699 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10700 */
10701HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10702{
10703 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10704 PVM pVM = pVCpu->CTX_SUFF(pVM);
10705 int rc = VINF_SUCCESS;
10706
10707 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10708 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10709 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10710 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10711 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10712 {
10713 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10714 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10715 }
10716 AssertRCReturn(rc, rc);
10717 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10718
10719 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10720 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10721 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10722
10723 if (RT_LIKELY(rc == VINF_SUCCESS))
10724 {
10725 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10726
10727 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10728 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10729 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10730 {
10731 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10732 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10733 EMInterpretWrmsr() changes it. */
10734 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10735 }
10736 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10737 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10738 else if (pMixedCtx->ecx == MSR_K6_EFER)
10739 {
10740 /*
10741 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10742 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10743 * the other bits as well, SCE and NXE. See @bugref{7368}.
10744 */
10745 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10746 }
10747
10748 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10749 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10750 {
10751 switch (pMixedCtx->ecx)
10752 {
10753 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10754 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10755 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10756 case MSR_K8_FS_BASE: /* no break */
10757 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10758 case MSR_K6_EFER: /* already handled above */ break;
10759 default:
10760 {
10761 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10762 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10763#if HC_ARCH_BITS == 64
10764 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10765 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10766#endif
10767 break;
10768 }
10769 }
10770 }
10771#ifdef VBOX_STRICT
10772 else
10773 {
10774 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10775 switch (pMixedCtx->ecx)
10776 {
10777 case MSR_IA32_SYSENTER_CS:
10778 case MSR_IA32_SYSENTER_EIP:
10779 case MSR_IA32_SYSENTER_ESP:
10780 case MSR_K8_FS_BASE:
10781 case MSR_K8_GS_BASE:
10782 {
10783 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10784 HMVMX_RETURN_UNEXPECTED_EXIT();
10785 }
10786
10787 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10788 default:
10789 {
10790 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10791 {
10792 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10793 if (pMixedCtx->ecx != MSR_K6_EFER)
10794 {
10795 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10796 pMixedCtx->ecx));
10797 HMVMX_RETURN_UNEXPECTED_EXIT();
10798 }
10799 }
10800
10801#if HC_ARCH_BITS == 64
10802 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10803 {
10804 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10805 HMVMX_RETURN_UNEXPECTED_EXIT();
10806 }
10807#endif
10808 break;
10809 }
10810 }
10811 }
10812#endif /* VBOX_STRICT */
10813 }
10814 return rc;
10815}
10816
10817
10818/**
10819 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10820 */
10821HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10822{
10823 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10824
10825 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10826 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10827 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10828 return VERR_EM_INTERPRETER;
10829 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10830 HMVMX_RETURN_UNEXPECTED_EXIT();
10831}
10832
10833
10834/**
10835 * VM-exit handler for when the TPR value is lowered below the specified
10836 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10837 */
10838HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10839{
10840 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10841 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10842
10843 /*
10844 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10845 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10846 * resume guest execution.
10847 */
10848 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10849 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10850 return VINF_SUCCESS;
10851}
10852
10853
10854/**
10855 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10856 * VM-exit.
10857 *
10858 * @retval VINF_SUCCESS when guest execution can continue.
10859 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10860 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10861 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10862 * recompiler.
10863 */
10864HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10865{
10866 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10867 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10868 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10869 AssertRCReturn(rc, rc);
10870
10871 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10872 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10873 PVM pVM = pVCpu->CTX_SUFF(pVM);
10874 switch (uAccessType)
10875 {
10876 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10877 {
10878#if 0
10879 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10880 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10881#else
10882 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10883 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10884 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10885#endif
10886 AssertRCReturn(rc, rc);
10887
10888 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10889 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10890 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10891 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10892
10893 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10894 {
10895 case 0: /* CR0 */
10896 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10897 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10898 break;
10899 case 2: /* CR2 */
10900 /* Nothing to do here, CR2 it's not part of the VMCS. */
10901 break;
10902 case 3: /* CR3 */
10903 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10904 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10905 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10906 break;
10907 case 4: /* CR4 */
10908 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10909 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10910 break;
10911 case 8: /* CR8 */
10912 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10913 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10914 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10915 break;
10916 default:
10917 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10918 break;
10919 }
10920
10921 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10922 break;
10923 }
10924
10925 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10926 {
10927 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10928 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10929 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10930 AssertRCReturn(rc, rc);
10931 Assert( !pVM->hm.s.fNestedPaging
10932 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10933 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10934
10935 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10936 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10937 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10938
10939 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10940 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10941 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10942 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10943 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10944 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10945 break;
10946 }
10947
10948 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10949 {
10950 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10951 AssertRCReturn(rc, rc);
10952 rc = EMInterpretCLTS(pVM, pVCpu);
10953 AssertRCReturn(rc, rc);
10954 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10955 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10956 Log4(("CRX CLTS write rc=%d\n", rc));
10957 break;
10958 }
10959
10960 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10961 {
10962 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10963 AssertRCReturn(rc, rc);
10964 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10965 if (RT_LIKELY(rc == VINF_SUCCESS))
10966 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10967 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10968 Log4(("CRX LMSW write rc=%d\n", rc));
10969 break;
10970 }
10971
10972 default:
10973 {
10974 AssertMsgFailed(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType));
10975 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10976 }
10977 }
10978
10979 /* Validate possible error codes. */
10980 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10981 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10982 if (RT_SUCCESS(rc))
10983 {
10984 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10985 AssertRCReturn(rc2, rc2);
10986 }
10987
10988 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10989 return rc;
10990}
10991
10992
10993/**
10994 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10995 * VM-exit.
10996 */
10997HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10998{
10999 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11000 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
11001
11002 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11003 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11004 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11005 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
11006 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
11007 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
11008 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
11009 AssertRCReturn(rc2, rc2);
11010
11011 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
11012 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
11013 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
11014 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
11015 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
11016 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
11017 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11018 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
11019
11020 /* I/O operation lookup arrays. */
11021 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
11022 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
11023
11024 VBOXSTRICTRC rcStrict;
11025 uint32_t const cbValue = s_aIOSizes[uIOWidth];
11026 uint32_t const cbInstr = pVmxTransient->cbInstr;
11027 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
11028 PVM pVM = pVCpu->CTX_SUFF(pVM);
11029 if (fIOString)
11030 {
11031#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
11032 /*
11033 * INS/OUTS - I/O String instruction.
11034 *
11035 * Use instruction-information if available, otherwise fall back on
11036 * interpreting the instruction.
11037 */
11038 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11039 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
11040 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
11041 {
11042 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11043 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11044 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11045 AssertRCReturn(rc2, rc2);
11046 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
11047 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
11048 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
11049 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
11050 if (fIOWrite)
11051 {
11052 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
11053 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
11054 }
11055 else
11056 {
11057 /*
11058 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
11059 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
11060 * See Intel Instruction spec. for "INS".
11061 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
11062 */
11063 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
11064 }
11065 }
11066 else
11067 {
11068 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11069 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11070 AssertRCReturn(rc2, rc2);
11071 rcStrict = IEMExecOne(pVCpu);
11072 }
11073 /** @todo IEM needs to be setting these flags somehow. */
11074 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11075 fUpdateRipAlready = true;
11076#else
11077 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11078 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
11079 if (RT_SUCCESS(rcStrict))
11080 {
11081 if (fIOWrite)
11082 {
11083 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11084 (DISCPUMODE)pDis->uAddrMode, cbValue);
11085 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
11086 }
11087 else
11088 {
11089 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11090 (DISCPUMODE)pDis->uAddrMode, cbValue);
11091 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11092 }
11093 }
11094 else
11095 {
11096 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
11097 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11098 }
11099#endif
11100 }
11101 else
11102 {
11103 /*
11104 * IN/OUT - I/O instruction.
11105 */
11106 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11107 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11108 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11109 if (fIOWrite)
11110 {
11111 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11112 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11113 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11114 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11115 }
11116 else
11117 {
11118 uint32_t u32Result = 0;
11119 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11120 if (IOM_SUCCESS(rcStrict))
11121 {
11122 /* Save result of I/O IN instr. in AL/AX/EAX. */
11123 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11124 }
11125 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11126 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11127 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11128 }
11129 }
11130
11131 if (IOM_SUCCESS(rcStrict))
11132 {
11133 if (!fUpdateRipAlready)
11134 {
11135 pMixedCtx->rip += cbInstr;
11136 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11137 }
11138
11139 /*
11140 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11141 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11142 */
11143 if (fIOString)
11144 {
11145 /** @todo Single-step for INS/OUTS with REP prefix? */
11146 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11147 }
11148 else if (fStepping)
11149 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11150
11151 /*
11152 * If any I/O breakpoints are armed, we need to check if one triggered
11153 * and take appropriate action.
11154 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11155 */
11156 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11157 AssertRCReturn(rc2, rc2);
11158
11159 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11160 * execution engines about whether hyper BPs and such are pending. */
11161 uint32_t const uDr7 = pMixedCtx->dr[7];
11162 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11163 && X86_DR7_ANY_RW_IO(uDr7)
11164 && (pMixedCtx->cr4 & X86_CR4_DE))
11165 || DBGFBpIsHwIoArmed(pVM)))
11166 {
11167 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11168
11169 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11170 VMMRZCallRing3Disable(pVCpu);
11171 HM_DISABLE_PREEMPT_IF_NEEDED();
11172
11173 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11174
11175 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11176 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11177 {
11178 /* Raise #DB. */
11179 if (fIsGuestDbgActive)
11180 ASMSetDR6(pMixedCtx->dr[6]);
11181 if (pMixedCtx->dr[7] != uDr7)
11182 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11183
11184 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11185 }
11186 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11187 else if ( rcStrict2 != VINF_SUCCESS
11188 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11189 rcStrict = rcStrict2;
11190
11191 HM_RESTORE_PREEMPT_IF_NEEDED();
11192 VMMRZCallRing3Enable(pVCpu);
11193 }
11194 }
11195
11196#ifdef DEBUG
11197 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11198 Assert(!fIOWrite);
11199 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11200 Assert(fIOWrite);
11201 else
11202 {
11203 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11204 * statuses, that the VMM device and some others may return. See
11205 * IOM_SUCCESS() for guidance. */
11206 AssertMsg( RT_FAILURE(rcStrict)
11207 || rcStrict == VINF_SUCCESS
11208 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11209 || rcStrict == VINF_EM_DBG_BREAKPOINT
11210 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11211 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11212 }
11213#endif
11214
11215 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11216 return VBOXSTRICTRC_TODO(rcStrict);
11217}
11218
11219
11220/**
11221 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11222 * VM-exit.
11223 */
11224HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11225{
11226 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11227
11228 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11229 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11230 AssertRCReturn(rc, rc);
11231 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11232 {
11233 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11234 AssertRCReturn(rc, rc);
11235 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11236 {
11237 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11238
11239 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11240 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11241
11242 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11243 Assert(!pVCpu->hm.s.Event.fPending);
11244 pVCpu->hm.s.Event.fPending = true;
11245 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11246 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11247 AssertRCReturn(rc, rc);
11248 if (fErrorCodeValid)
11249 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11250 else
11251 pVCpu->hm.s.Event.u32ErrCode = 0;
11252 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11253 && uVector == X86_XCPT_PF)
11254 {
11255 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11256 }
11257
11258 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11259 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11260 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11261 }
11262 }
11263
11264 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11265 * emulation. */
11266 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11267 return VERR_EM_INTERPRETER;
11268}
11269
11270
11271/**
11272 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11273 */
11274HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11275{
11276 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11277 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11278 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11279 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11280 AssertRCReturn(rc, rc);
11281 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11282 return VINF_EM_DBG_STEPPED;
11283}
11284
11285
11286/**
11287 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11288 */
11289HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11290{
11291 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11292
11293 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11294 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11295 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11296 {
11297 if (rc == VINF_HM_DOUBLE_FAULT)
11298 rc = VINF_SUCCESS;
11299 return rc;
11300 }
11301
11302#if 0
11303 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11304 * just sync the whole thing. */
11305 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11306#else
11307 /* Aggressive state sync. for now. */
11308 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11309 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11310 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11311#endif
11312 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11313 AssertRCReturn(rc, rc);
11314
11315 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11316 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11317 switch (uAccessType)
11318 {
11319 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11320 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11321 {
11322 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11323 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11324 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11325
11326 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11327 GCPhys &= PAGE_BASE_GC_MASK;
11328 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11329 PVM pVM = pVCpu->CTX_SUFF(pVM);
11330 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11331 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11332
11333 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11334 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11335 CPUMCTX2CORE(pMixedCtx), GCPhys);
11336 rc = VBOXSTRICTRC_VAL(rc2);
11337 Log4(("ApicAccess rc=%d\n", rc));
11338 if ( rc == VINF_SUCCESS
11339 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11340 || rc == VERR_PAGE_NOT_PRESENT)
11341 {
11342 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11343 | HM_CHANGED_GUEST_RSP
11344 | HM_CHANGED_GUEST_RFLAGS
11345 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11346 rc = VINF_SUCCESS;
11347 }
11348 break;
11349 }
11350
11351 default:
11352 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11353 rc = VINF_EM_RAW_EMULATE_INSTR;
11354 break;
11355 }
11356
11357 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11358 if (rc != VINF_SUCCESS)
11359 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccessToR3);
11360 return rc;
11361}
11362
11363
11364/**
11365 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11366 * VM-exit.
11367 */
11368HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11369{
11370 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11371
11372 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11373 if (pVmxTransient->fWasGuestDebugStateActive)
11374 {
11375 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11376 HMVMX_RETURN_UNEXPECTED_EXIT();
11377 }
11378
11379 int rc = VERR_INTERNAL_ERROR_5;
11380 if ( !DBGFIsStepping(pVCpu)
11381 && !pVCpu->hm.s.fSingleInstruction
11382 && !pVmxTransient->fWasHyperDebugStateActive)
11383 {
11384 /* Don't intercept MOV DRx and #DB any more. */
11385 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11386 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11387 AssertRCReturn(rc, rc);
11388
11389 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11390 {
11391#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11392 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11393 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11394 AssertRCReturn(rc, rc);
11395#endif
11396 }
11397
11398 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11399 VMMRZCallRing3Disable(pVCpu);
11400 HM_DISABLE_PREEMPT_IF_NEEDED();
11401
11402 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11403 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11404 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11405
11406 HM_RESTORE_PREEMPT_IF_NEEDED();
11407 VMMRZCallRing3Enable(pVCpu);
11408
11409#ifdef VBOX_WITH_STATISTICS
11410 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11411 AssertRCReturn(rc, rc);
11412 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11413 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11414 else
11415 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11416#endif
11417 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11418 return VINF_SUCCESS;
11419 }
11420
11421 /*
11422 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11423 * Update the segment registers and DR7 from the CPU.
11424 */
11425 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11426 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11427 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11428 AssertRCReturn(rc, rc);
11429 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11430
11431 PVM pVM = pVCpu->CTX_SUFF(pVM);
11432 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11433 {
11434 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11435 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11436 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11437 if (RT_SUCCESS(rc))
11438 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11439 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11440 }
11441 else
11442 {
11443 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11444 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11445 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11446 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11447 }
11448
11449 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11450 if (RT_SUCCESS(rc))
11451 {
11452 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11453 AssertRCReturn(rc2, rc2);
11454 }
11455 return rc;
11456}
11457
11458
11459/**
11460 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11461 * Conditional VM-exit.
11462 */
11463HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11464{
11465 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11466 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11467
11468 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11469 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11470 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11471 {
11472 if (rc == VINF_HM_DOUBLE_FAULT)
11473 rc = VINF_SUCCESS;
11474 return rc;
11475 }
11476
11477 RTGCPHYS GCPhys = 0;
11478 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11479
11480#if 0
11481 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11482#else
11483 /* Aggressive state sync. for now. */
11484 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11485 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11486 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11487#endif
11488 AssertRCReturn(rc, rc);
11489
11490 /*
11491 * If we succeed, resume guest execution.
11492 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11493 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11494 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11495 * weird case. See @bugref{6043}.
11496 */
11497 PVM pVM = pVCpu->CTX_SUFF(pVM);
11498 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11499 rc = VBOXSTRICTRC_VAL(rc2);
11500 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11501 if ( rc == VINF_SUCCESS
11502 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11503 || rc == VERR_PAGE_NOT_PRESENT)
11504 {
11505 /* Successfully handled MMIO operation. */
11506 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11507 | HM_CHANGED_GUEST_RSP
11508 | HM_CHANGED_GUEST_RFLAGS
11509 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11510 rc = VINF_SUCCESS;
11511 }
11512 return rc;
11513}
11514
11515
11516/**
11517 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11518 * VM-exit.
11519 */
11520HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11521{
11522 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11523 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11524
11525 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11526 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11527 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11528 {
11529 if (rc == VINF_HM_DOUBLE_FAULT)
11530 rc = VINF_SUCCESS;
11531 return rc;
11532 }
11533
11534 RTGCPHYS GCPhys = 0;
11535 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11536 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11537#if 0
11538 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11539#else
11540 /* Aggressive state sync. for now. */
11541 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11542 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11543 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11544#endif
11545 AssertRCReturn(rc, rc);
11546
11547 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11548 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11549
11550 RTGCUINT uErrorCode = 0;
11551 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11552 uErrorCode |= X86_TRAP_PF_ID;
11553 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11554 uErrorCode |= X86_TRAP_PF_RW;
11555 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11556 uErrorCode |= X86_TRAP_PF_P;
11557
11558 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11559
11560 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11561 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11562
11563 /* Handle the pagefault trap for the nested shadow table. */
11564 PVM pVM = pVCpu->CTX_SUFF(pVM);
11565 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11566 TRPMResetTrap(pVCpu);
11567
11568 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11569 if ( rc == VINF_SUCCESS
11570 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11571 || rc == VERR_PAGE_NOT_PRESENT)
11572 {
11573 /* Successfully synced our nested page tables. */
11574 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11575 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11576 | HM_CHANGED_GUEST_RSP
11577 | HM_CHANGED_GUEST_RFLAGS);
11578 return VINF_SUCCESS;
11579 }
11580
11581 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11582 return rc;
11583}
11584
11585/** @} */
11586
11587/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11588/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11589/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11590
11591/** @name VM-exit exception handlers.
11592 * @{
11593 */
11594
11595/**
11596 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11597 */
11598static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11599{
11600 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11601 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11602
11603 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11604 AssertRCReturn(rc, rc);
11605
11606 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11607 {
11608 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11609 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11610
11611 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11612 * provides VM-exit instruction length. If this causes problem later,
11613 * disassemble the instruction like it's done on AMD-V. */
11614 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11615 AssertRCReturn(rc2, rc2);
11616 return rc;
11617 }
11618
11619 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11620 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11621 return rc;
11622}
11623
11624
11625/**
11626 * VM-exit exception handler for #BP (Breakpoint exception).
11627 */
11628static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11629{
11630 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11631 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11632
11633 /** @todo Try optimize this by not saving the entire guest state unless
11634 * really needed. */
11635 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11636 AssertRCReturn(rc, rc);
11637
11638 PVM pVM = pVCpu->CTX_SUFF(pVM);
11639 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11640 if (rc == VINF_EM_RAW_GUEST_TRAP)
11641 {
11642 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11643 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11644 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11645 AssertRCReturn(rc, rc);
11646
11647 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11648 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11649 }
11650
11651 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11652 return rc;
11653}
11654
11655
11656/**
11657 * VM-exit exception handler for #DB (Debug exception).
11658 */
11659static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11660{
11661 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11662 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11663 Log6(("XcptDB\n"));
11664
11665 /*
11666 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11667 * for processing.
11668 */
11669 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11670 AssertRCReturn(rc, rc);
11671
11672 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11673 uint64_t uDR6 = X86_DR6_INIT_VAL;
11674 uDR6 |= ( pVmxTransient->uExitQualification
11675 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11676
11677 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11678 if (rc == VINF_EM_RAW_GUEST_TRAP)
11679 {
11680 /*
11681 * The exception was for the guest. Update DR6, DR7.GD and
11682 * IA32_DEBUGCTL.LBR before forwarding it.
11683 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11684 */
11685 VMMRZCallRing3Disable(pVCpu);
11686 HM_DISABLE_PREEMPT_IF_NEEDED();
11687
11688 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11689 pMixedCtx->dr[6] |= uDR6;
11690 if (CPUMIsGuestDebugStateActive(pVCpu))
11691 ASMSetDR6(pMixedCtx->dr[6]);
11692
11693 HM_RESTORE_PREEMPT_IF_NEEDED();
11694 VMMRZCallRing3Enable(pVCpu);
11695
11696 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11697 AssertRCReturn(rc, rc);
11698
11699 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11700 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11701
11702 /* Paranoia. */
11703 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11704 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11705
11706 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11707 AssertRCReturn(rc, rc);
11708
11709 /*
11710 * Raise #DB in the guest.
11711 *
11712 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11713 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11714 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11715 *
11716 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11717 */
11718 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11719 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11720 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11721 AssertRCReturn(rc, rc);
11722 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11723 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11724 return VINF_SUCCESS;
11725 }
11726
11727 /*
11728 * Not a guest trap, must be a hypervisor related debug event then.
11729 * Update DR6 in case someone is interested in it.
11730 */
11731 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11732 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11733 CPUMSetHyperDR6(pVCpu, uDR6);
11734
11735 return rc;
11736}
11737
11738
11739/**
11740 * VM-exit exception handler for #NM (Device-not-available exception: floating
11741 * point exception).
11742 */
11743static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11744{
11745 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11746
11747 /* We require CR0 and EFER. EFER is always up-to-date. */
11748 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11749 AssertRCReturn(rc, rc);
11750
11751 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11752 VMMRZCallRing3Disable(pVCpu);
11753 HM_DISABLE_PREEMPT_IF_NEEDED();
11754
11755 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11756 if (pVmxTransient->fWasGuestFPUStateActive)
11757 {
11758 rc = VINF_EM_RAW_GUEST_TRAP;
11759 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11760 }
11761 else
11762 {
11763#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11764 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11765#endif
11766 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11767 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11768 }
11769
11770 HM_RESTORE_PREEMPT_IF_NEEDED();
11771 VMMRZCallRing3Enable(pVCpu);
11772
11773 if (rc == VINF_SUCCESS)
11774 {
11775 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11776 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11777 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11778 pVCpu->hm.s.fUseGuestFpu = true;
11779 }
11780 else
11781 {
11782 /* Forward #NM to the guest. */
11783 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11784 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11785 AssertRCReturn(rc, rc);
11786 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11787 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11788 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11789 }
11790
11791 return VINF_SUCCESS;
11792}
11793
11794
11795/**
11796 * VM-exit exception handler for #GP (General-protection exception).
11797 *
11798 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11799 */
11800static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11801{
11802 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11803 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11804
11805 int rc = VERR_INTERNAL_ERROR_5;
11806 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11807 {
11808#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11809 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11810 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11811 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11812 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11813 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11814 AssertRCReturn(rc, rc);
11815 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11816 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11817 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11818 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11819 return rc;
11820#else
11821 /* We don't intercept #GP. */
11822 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11823 NOREF(pVmxTransient);
11824 return VERR_VMX_UNEXPECTED_EXCEPTION;
11825#endif
11826 }
11827
11828 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11829 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11830
11831 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11832 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11833 AssertRCReturn(rc, rc);
11834
11835 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11836 uint32_t cbOp = 0;
11837 PVM pVM = pVCpu->CTX_SUFF(pVM);
11838 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11839 if (RT_SUCCESS(rc))
11840 {
11841 rc = VINF_SUCCESS;
11842 Assert(cbOp == pDis->cbInstr);
11843 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11844 switch (pDis->pCurInstr->uOpcode)
11845 {
11846 case OP_CLI:
11847 {
11848 pMixedCtx->eflags.Bits.u1IF = 0;
11849 pMixedCtx->eflags.Bits.u1RF = 0;
11850 pMixedCtx->rip += pDis->cbInstr;
11851 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11852 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11853 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11854 break;
11855 }
11856
11857 case OP_STI:
11858 {
11859 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11860 pMixedCtx->eflags.Bits.u1IF = 1;
11861 pMixedCtx->eflags.Bits.u1RF = 0;
11862 pMixedCtx->rip += pDis->cbInstr;
11863 if (!fOldIF)
11864 {
11865 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11866 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11867 }
11868 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11869 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11870 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11871 break;
11872 }
11873
11874 case OP_HLT:
11875 {
11876 rc = VINF_EM_HALT;
11877 pMixedCtx->rip += pDis->cbInstr;
11878 pMixedCtx->eflags.Bits.u1RF = 0;
11879 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11880 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11881 break;
11882 }
11883
11884 case OP_POPF:
11885 {
11886 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11887 uint32_t cbParm;
11888 uint32_t uMask;
11889 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11890 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11891 {
11892 cbParm = 4;
11893 uMask = 0xffffffff;
11894 }
11895 else
11896 {
11897 cbParm = 2;
11898 uMask = 0xffff;
11899 }
11900
11901 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11902 RTGCPTR GCPtrStack = 0;
11903 X86EFLAGS Eflags;
11904 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11905 &GCPtrStack);
11906 if (RT_SUCCESS(rc))
11907 {
11908 Assert(sizeof(Eflags.u32) >= cbParm);
11909 Eflags.u32 = 0;
11910 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11911 }
11912 if (RT_FAILURE(rc))
11913 {
11914 rc = VERR_EM_INTERPRETER;
11915 break;
11916 }
11917 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11918 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
11919 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11920 pMixedCtx->esp += cbParm;
11921 pMixedCtx->esp &= uMask;
11922 pMixedCtx->rip += pDis->cbInstr;
11923 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11924 | HM_CHANGED_GUEST_RSP
11925 | HM_CHANGED_GUEST_RFLAGS);
11926 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11927 if (fStepping)
11928 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11929
11930 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11931 break;
11932 }
11933
11934 case OP_PUSHF:
11935 {
11936 uint32_t cbParm;
11937 uint32_t uMask;
11938 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11939 {
11940 cbParm = 4;
11941 uMask = 0xffffffff;
11942 }
11943 else
11944 {
11945 cbParm = 2;
11946 uMask = 0xffff;
11947 }
11948
11949 /* Get the stack pointer & push the contents of eflags onto the stack. */
11950 RTGCPTR GCPtrStack = 0;
11951 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11952 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11953 if (RT_FAILURE(rc))
11954 {
11955 rc = VERR_EM_INTERPRETER;
11956 break;
11957 }
11958 X86EFLAGS Eflags = pMixedCtx->eflags;
11959 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11960 Eflags.Bits.u1RF = 0;
11961 Eflags.Bits.u1VM = 0;
11962
11963 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11964 if (RT_FAILURE(rc))
11965 {
11966 rc = VERR_EM_INTERPRETER;
11967 break;
11968 }
11969 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11970 pMixedCtx->esp -= cbParm;
11971 pMixedCtx->esp &= uMask;
11972 pMixedCtx->rip += pDis->cbInstr;
11973 pMixedCtx->eflags.Bits.u1RF = 0;
11974 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11975 | HM_CHANGED_GUEST_RSP
11976 | HM_CHANGED_GUEST_RFLAGS);
11977 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11978 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11979 break;
11980 }
11981
11982 case OP_IRET:
11983 {
11984 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11985 * instruction reference. */
11986 RTGCPTR GCPtrStack = 0;
11987 uint32_t uMask = 0xffff;
11988 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11989 uint16_t aIretFrame[3];
11990 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11991 {
11992 rc = VERR_EM_INTERPRETER;
11993 break;
11994 }
11995 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11996 &GCPtrStack);
11997 if (RT_SUCCESS(rc))
11998 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11999 if (RT_FAILURE(rc))
12000 {
12001 rc = VERR_EM_INTERPRETER;
12002 break;
12003 }
12004 pMixedCtx->eip = 0;
12005 pMixedCtx->ip = aIretFrame[0];
12006 pMixedCtx->cs.Sel = aIretFrame[1];
12007 pMixedCtx->cs.ValidSel = aIretFrame[1];
12008 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
12009 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
12010 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
12011 pMixedCtx->sp += sizeof(aIretFrame);
12012 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12013 | HM_CHANGED_GUEST_SEGMENT_REGS
12014 | HM_CHANGED_GUEST_RSP
12015 | HM_CHANGED_GUEST_RFLAGS);
12016 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
12017 if (fStepping)
12018 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
12019 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
12020 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
12021 break;
12022 }
12023
12024 case OP_INT:
12025 {
12026 uint16_t uVector = pDis->Param1.uValue & 0xff;
12027 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
12028 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12029 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12030 break;
12031 }
12032
12033 case OP_INTO:
12034 {
12035 if (pMixedCtx->eflags.Bits.u1OF)
12036 {
12037 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
12038 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12039 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12040 }
12041 else
12042 {
12043 pMixedCtx->eflags.Bits.u1RF = 0;
12044 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12045 }
12046 break;
12047 }
12048
12049 default:
12050 {
12051 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
12052 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
12053 EMCODETYPE_SUPERVISOR);
12054 rc = VBOXSTRICTRC_VAL(rc2);
12055 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
12056 /** @todo We have to set pending-debug exceptions here when the guest is
12057 * single-stepping depending on the instruction that was interpreted. */
12058 Log4(("#GP rc=%Rrc\n", rc));
12059 break;
12060 }
12061 }
12062 }
12063 else
12064 rc = VERR_EM_INTERPRETER;
12065
12066 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
12067 ("#GP Unexpected rc=%Rrc\n", rc));
12068 return rc;
12069}
12070
12071
12072#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12073/**
12074 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
12075 * the exception reported in the VMX transient structure back into the VM.
12076 *
12077 * @remarks Requires uExitIntInfo in the VMX transient structure to be
12078 * up-to-date.
12079 */
12080static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12081{
12082 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12083
12084 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12085 hmR0VmxCheckExitDueToEventDelivery(). */
12086 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12087 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12088 AssertRCReturn(rc, rc);
12089 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12090
12091#ifdef DEBUG_ramshankar
12092 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12093 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12094 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12095#endif
12096
12097 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12098 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12099 return VINF_SUCCESS;
12100}
12101#endif
12102
12103
12104/**
12105 * VM-exit exception handler for #PF (Page-fault exception).
12106 */
12107static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12108{
12109 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12110 PVM pVM = pVCpu->CTX_SUFF(pVM);
12111 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12112 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12113 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12114 AssertRCReturn(rc, rc);
12115
12116#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12117 if (pVM->hm.s.fNestedPaging)
12118 {
12119 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12120 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12121 {
12122 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12123 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12124 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12125 }
12126 else
12127 {
12128 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12129 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12130 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12131 }
12132 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12133 return rc;
12134 }
12135#else
12136 Assert(!pVM->hm.s.fNestedPaging);
12137 NOREF(pVM);
12138#endif
12139
12140 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12141 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12142 if (pVmxTransient->fVectoringPF)
12143 {
12144 Assert(pVCpu->hm.s.Event.fPending);
12145 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12146 }
12147
12148 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12149 AssertRCReturn(rc, rc);
12150
12151 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12152 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12153
12154 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12155 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12156 (RTGCPTR)pVmxTransient->uExitQualification);
12157
12158 Log4(("#PF: rc=%Rrc\n", rc));
12159 if (rc == VINF_SUCCESS)
12160 {
12161 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12162 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12163 * memory? We don't update the whole state here... */
12164 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12165 | HM_CHANGED_GUEST_RSP
12166 | HM_CHANGED_GUEST_RFLAGS
12167 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12168 TRPMResetTrap(pVCpu);
12169 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12170 return rc;
12171 }
12172
12173 if (rc == VINF_EM_RAW_GUEST_TRAP)
12174 {
12175 if (!pVmxTransient->fVectoringDoublePF)
12176 {
12177 /* It's a guest page fault and needs to be reflected to the guest. */
12178 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12179 TRPMResetTrap(pVCpu);
12180 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12181 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12182 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12183 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12184 }
12185 else
12186 {
12187 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12188 TRPMResetTrap(pVCpu);
12189 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12190 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12191 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12192 }
12193
12194 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12195 return VINF_SUCCESS;
12196 }
12197
12198 TRPMResetTrap(pVCpu);
12199 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12200 return rc;
12201}
12202
12203/** @} */
12204
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette