VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 55129

Last change on this file since 55129 was 55129, checked in by vboxsync, 10 years ago

VMM/GIM: Allow dynamic enabling of #UD traps and per-VCPU hypercalls.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 508.4 KB
Line 
1/* $Id: HMVMXR0.cpp 55129 2015-04-08 11:31:47Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39# include "dtrace/VBoxVMM.h"
40
41#ifdef DEBUG_ramshankar
42# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
43# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
44# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_CHECK_GUEST_STATE
46# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
47# define HMVMX_ALWAYS_TRAP_PF
48# define HMVMX_ALWAYS_SWAP_FPU_STATE
49# define HMVMX_ALWAYS_FLUSH_TLB
50# define HMVMX_ALWAYS_SWAP_EFER
51#endif
52
53
54/*******************************************************************************
55* Defined Constants And Macros *
56*******************************************************************************/
57#if defined(RT_ARCH_AMD64)
58# define HMVMX_IS_64BIT_HOST_MODE() (true)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
61extern "C" uint32_t g_fVMXIs64bitHost;
62# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
63typedef uint64_t HMVMXHCUINTREG;
64#else
65# define HMVMX_IS_64BIT_HOST_MODE() (false)
66typedef RTHCUINTREG HMVMXHCUINTREG;
67#endif
68
69/** Use the function table. */
70#define HMVMX_USE_FUNCTION_TABLE
71
72/** Determine which tagged-TLB flush handler to use. */
73#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
74#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
75#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
76#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
77
78/** @name Updated-guest-state flags.
79 * @{ */
80#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
81#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
82#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
83#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
84#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
85#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
86#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
87#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
88#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
89#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
90#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
91#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
92#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
93#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
94#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
95#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
96#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
97#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
98#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
99#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
100#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
101 | HMVMX_UPDATED_GUEST_RSP \
102 | HMVMX_UPDATED_GUEST_RFLAGS \
103 | HMVMX_UPDATED_GUEST_CR0 \
104 | HMVMX_UPDATED_GUEST_CR3 \
105 | HMVMX_UPDATED_GUEST_CR4 \
106 | HMVMX_UPDATED_GUEST_GDTR \
107 | HMVMX_UPDATED_GUEST_IDTR \
108 | HMVMX_UPDATED_GUEST_LDTR \
109 | HMVMX_UPDATED_GUEST_TR \
110 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
111 | HMVMX_UPDATED_GUEST_DEBUG \
112 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
113 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
114 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
115 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
116 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
117 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
118 | HMVMX_UPDATED_GUEST_INTR_STATE \
119 | HMVMX_UPDATED_GUEST_APIC_STATE)
120/** @} */
121
122/** @name
123 * Flags to skip redundant reads of some common VMCS fields that are not part of
124 * the guest-CPU state but are in the transient structure.
125 */
126#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
127#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
131#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
132#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
133/** @} */
134
135/** @name
136 * States of the VMCS.
137 *
138 * This does not reflect all possible VMCS states but currently only those
139 * needed for maintaining the VMCS consistently even when thread-context hooks
140 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
141 */
142#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
143#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
144#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
145/** @} */
146
147/**
148 * Exception bitmap mask for real-mode guests (real-on-v86).
149 *
150 * We need to intercept all exceptions manually except:
151 * - #NM, #MF handled in hmR0VmxLoadSharedCR0().
152 * - #DB handled in hmR0VmxLoadSharedDebugState().
153 * - #PF need not be intercepted even in real-mode if we have Nested Paging
154 * support.
155 */
156#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
157 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
158 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
159 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
160 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
161 /* RT_BIT(X86_XCPT_MF) */ | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
162 | RT_BIT(X86_XCPT_XF))
163
164/**
165 * Exception bitmap mask for all contributory exceptions.
166 *
167 * Page fault is deliberately excluded here as it's conditional as to whether
168 * it's contributory or benign. Page faults are handled separately.
169 */
170#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
171 | RT_BIT(X86_XCPT_DE))
172
173/** Maximum VM-instruction error number. */
174#define HMVMX_INSTR_ERROR_MAX 28
175
176/** Profiling macro. */
177#ifdef HM_PROFILE_EXIT_DISPATCH
178# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
179# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
180#else
181# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
182# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
183#endif
184
185/** Assert that preemption is disabled or covered by thread-context hooks. */
186#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
187 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
188
189/** Assert that we haven't migrated CPUs when thread-context hooks are not
190 * used. */
191#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
192 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
193 ("Illegal migration! Entered on CPU %u Current %u\n", \
194 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
195
196/** Helper macro for VM-exit handlers called unexpectedly. */
197#define HMVMX_RETURN_UNEXPECTED_EXIT() \
198 do { \
199 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
200 return VERR_VMX_UNEXPECTED_EXIT; \
201 } while (0)
202
203
204/*******************************************************************************
205* Structures and Typedefs *
206*******************************************************************************/
207/**
208 * VMX transient state.
209 *
210 * A state structure for holding miscellaneous information across
211 * VMX non-root operation and restored after the transition.
212 */
213typedef struct VMXTRANSIENT
214{
215 /** The host's rflags/eflags. */
216 RTCCUINTREG uEflags;
217#if HC_ARCH_BITS == 32
218 uint32_t u32Alignment0;
219#endif
220 /** The guest's TPR value used for TPR shadowing. */
221 uint8_t u8GuestTpr;
222 /** Alignment. */
223 uint8_t abAlignment0[7];
224
225 /** The basic VM-exit reason. */
226 uint16_t uExitReason;
227 /** Alignment. */
228 uint16_t u16Alignment0;
229 /** The VM-exit interruption error code. */
230 uint32_t uExitIntErrorCode;
231 /** The VM-exit exit code qualification. */
232 uint64_t uExitQualification;
233
234 /** The VM-exit interruption-information field. */
235 uint32_t uExitIntInfo;
236 /** The VM-exit instruction-length field. */
237 uint32_t cbInstr;
238 /** The VM-exit instruction-information field. */
239 union
240 {
241 /** Plain unsigned int representation. */
242 uint32_t u;
243 /** INS and OUTS information. */
244 struct
245 {
246 uint32_t u6Reserved0 : 7;
247 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
248 uint32_t u3AddrSize : 3;
249 uint32_t u5Reserved1 : 5;
250 /** The segment register (X86_SREG_XXX). */
251 uint32_t iSegReg : 3;
252 uint32_t uReserved2 : 14;
253 } StrIo;
254 } ExitInstrInfo;
255 /** Whether the VM-entry failed or not. */
256 bool fVMEntryFailed;
257 /** Alignment. */
258 uint8_t abAlignment1[3];
259
260 /** The VM-entry interruption-information field. */
261 uint32_t uEntryIntInfo;
262 /** The VM-entry exception error code field. */
263 uint32_t uEntryXcptErrorCode;
264 /** The VM-entry instruction length field. */
265 uint32_t cbEntryInstr;
266
267 /** IDT-vectoring information field. */
268 uint32_t uIdtVectoringInfo;
269 /** IDT-vectoring error code. */
270 uint32_t uIdtVectoringErrorCode;
271
272 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
273 uint32_t fVmcsFieldsRead;
274
275 /** Whether the guest FPU was active at the time of VM-exit. */
276 bool fWasGuestFPUStateActive;
277 /** Whether the guest debug state was active at the time of VM-exit. */
278 bool fWasGuestDebugStateActive;
279 /** Whether the hyper debug state was active at the time of VM-exit. */
280 bool fWasHyperDebugStateActive;
281 /** Whether TSC-offsetting should be setup before VM-entry. */
282 bool fUpdateTscOffsettingAndPreemptTimer;
283 /** Whether the VM-exit was caused by a page-fault during delivery of a
284 * contributory exception or a page-fault. */
285 bool fVectoringDoublePF;
286 /** Whether the VM-exit was caused by a page-fault during delivery of an
287 * external interrupt or NMI. */
288 bool fVectoringPF;
289} VMXTRANSIENT;
290AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
291AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
292AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
294AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
295/** Pointer to VMX transient state. */
296typedef VMXTRANSIENT *PVMXTRANSIENT;
297
298
299/**
300 * MSR-bitmap read permissions.
301 */
302typedef enum VMXMSREXITREAD
303{
304 /** Reading this MSR causes a VM-exit. */
305 VMXMSREXIT_INTERCEPT_READ = 0xb,
306 /** Reading this MSR does not cause a VM-exit. */
307 VMXMSREXIT_PASSTHRU_READ
308} VMXMSREXITREAD;
309/** Pointer to MSR-bitmap read permissions. */
310typedef VMXMSREXITREAD* PVMXMSREXITREAD;
311
312/**
313 * MSR-bitmap write permissions.
314 */
315typedef enum VMXMSREXITWRITE
316{
317 /** Writing to this MSR causes a VM-exit. */
318 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
319 /** Writing to this MSR does not cause a VM-exit. */
320 VMXMSREXIT_PASSTHRU_WRITE
321} VMXMSREXITWRITE;
322/** Pointer to MSR-bitmap write permissions. */
323typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
324
325
326/**
327 * VMX VM-exit handler.
328 *
329 * @returns VBox status code.
330 * @param pVCpu Pointer to the VMCPU.
331 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
332 * out-of-sync. Make sure to update the required
333 * fields before using them.
334 * @param pVmxTransient Pointer to the VMX-transient structure.
335 */
336#ifndef HMVMX_USE_FUNCTION_TABLE
337typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
338#else
339typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
340/** Pointer to VM-exit handler. */
341typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342#endif
343
344
345/*******************************************************************************
346* Internal Functions *
347*******************************************************************************/
348static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
349static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
350static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
351 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
352 bool fStepping, uint32_t *puIntState);
353#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
354static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
355#endif
356#ifndef HMVMX_USE_FUNCTION_TABLE
357DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
358# define HMVMX_EXIT_DECL static int
359#else
360# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
361#endif
362DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
363 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
364
365/** @name VM-exit handlers.
366 * @{
367 */
368static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
369static FNVMXEXITHANDLER hmR0VmxExitExtInt;
370static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
371static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
372static FNVMXEXITHANDLER hmR0VmxExitSipi;
373static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
374static FNVMXEXITHANDLER hmR0VmxExitSmi;
375static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
376static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
377static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
378static FNVMXEXITHANDLER hmR0VmxExitCpuid;
379static FNVMXEXITHANDLER hmR0VmxExitGetsec;
380static FNVMXEXITHANDLER hmR0VmxExitHlt;
381static FNVMXEXITHANDLER hmR0VmxExitInvd;
382static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
383static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
384static FNVMXEXITHANDLER hmR0VmxExitVmcall;
385static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
386static FNVMXEXITHANDLER hmR0VmxExitRsm;
387static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
388static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
389static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
390static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
391static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
392static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
393static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
394static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
395static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
396static FNVMXEXITHANDLER hmR0VmxExitMwait;
397static FNVMXEXITHANDLER hmR0VmxExitMtf;
398static FNVMXEXITHANDLER hmR0VmxExitMonitor;
399static FNVMXEXITHANDLER hmR0VmxExitPause;
400static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
401static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
402static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
403static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
404static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
405static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
406static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
407static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
408static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
409static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
410static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
411static FNVMXEXITHANDLER hmR0VmxExitRdrand;
412static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
413/** @} */
414
415static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
417static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
418static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
419static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
420static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
421#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
422static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
423#endif
424static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
425
426/*******************************************************************************
427* Global Variables *
428*******************************************************************************/
429#ifdef HMVMX_USE_FUNCTION_TABLE
430
431/**
432 * VMX_EXIT dispatch table.
433 */
434static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
435{
436 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
437 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
438 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
439 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
440 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
441 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
442 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
443 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
444 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
445 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
446 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
447 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
448 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
449 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
450 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
451 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
452 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
453 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
454 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
455 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
456 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
457 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
458 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
459 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
460 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
461 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
462 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
463 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
464 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
465 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
466 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
467 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
468 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
469 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
470 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
471 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
472 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
473 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
474 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
475 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
476 /* 40 UNDEFINED */ hmR0VmxExitPause,
477 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
478 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
479 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
480 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
481 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
482 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
483 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
484 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
485 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
486 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
487 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
488 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
489 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
490 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
491 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
492 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
493 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
494 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
495 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
496};
497#endif /* HMVMX_USE_FUNCTION_TABLE */
498
499#ifdef VBOX_STRICT
500static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
501{
502 /* 0 */ "(Not Used)",
503 /* 1 */ "VMCALL executed in VMX root operation.",
504 /* 2 */ "VMCLEAR with invalid physical address.",
505 /* 3 */ "VMCLEAR with VMXON pointer.",
506 /* 4 */ "VMLAUNCH with non-clear VMCS.",
507 /* 5 */ "VMRESUME with non-launched VMCS.",
508 /* 6 */ "VMRESUME after VMXOFF",
509 /* 7 */ "VM-entry with invalid control fields.",
510 /* 8 */ "VM-entry with invalid host state fields.",
511 /* 9 */ "VMPTRLD with invalid physical address.",
512 /* 10 */ "VMPTRLD with VMXON pointer.",
513 /* 11 */ "VMPTRLD with incorrect revision identifier.",
514 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
515 /* 13 */ "VMWRITE to read-only VMCS component.",
516 /* 14 */ "(Not Used)",
517 /* 15 */ "VMXON executed in VMX root operation.",
518 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
519 /* 17 */ "VM-entry with non-launched executing VMCS.",
520 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
521 /* 19 */ "VMCALL with non-clear VMCS.",
522 /* 20 */ "VMCALL with invalid VM-exit control fields.",
523 /* 21 */ "(Not Used)",
524 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
525 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
526 /* 24 */ "VMCALL with invalid SMM-monitor features.",
527 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
528 /* 26 */ "VM-entry with events blocked by MOV SS.",
529 /* 27 */ "(Not Used)",
530 /* 28 */ "Invalid operand to INVEPT/INVVPID."
531};
532#endif /* VBOX_STRICT */
533
534
535
536/**
537 * Updates the VM's last error record. If there was a VMX instruction error,
538 * reads the error data from the VMCS and updates VCPU's last error record as
539 * well.
540 *
541 * @param pVM Pointer to the VM.
542 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
543 * VERR_VMX_UNABLE_TO_START_VM or
544 * VERR_VMX_INVALID_VMCS_FIELD).
545 * @param rc The error code.
546 */
547static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
548{
549 AssertPtr(pVM);
550 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
551 || rc == VERR_VMX_UNABLE_TO_START_VM)
552 {
553 AssertPtrReturnVoid(pVCpu);
554 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
555 }
556 pVM->hm.s.lLastError = rc;
557}
558
559
560/**
561 * Reads the VM-entry interruption-information field from the VMCS into the VMX
562 * transient structure.
563 *
564 * @returns VBox status code.
565 * @param pVmxTransient Pointer to the VMX transient structure.
566 *
567 * @remarks No-long-jump zone!!!
568 */
569DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
570{
571 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
572 AssertRCReturn(rc, rc);
573 return VINF_SUCCESS;
574}
575
576
577/**
578 * Reads the VM-entry exception error code field from the VMCS into
579 * the VMX transient structure.
580 *
581 * @returns VBox status code.
582 * @param pVmxTransient Pointer to the VMX transient structure.
583 *
584 * @remarks No-long-jump zone!!!
585 */
586DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
587{
588 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
589 AssertRCReturn(rc, rc);
590 return VINF_SUCCESS;
591}
592
593
594/**
595 * Reads the VM-entry exception error code field from the VMCS into
596 * the VMX transient structure.
597 *
598 * @returns VBox status code.
599 * @param pVmxTransient Pointer to the VMX transient structure.
600 *
601 * @remarks No-long-jump zone!!!
602 */
603DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
604{
605 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
606 AssertRCReturn(rc, rc);
607 return VINF_SUCCESS;
608}
609
610
611/**
612 * Reads the VM-exit interruption-information field from the VMCS into the VMX
613 * transient structure.
614 *
615 * @returns VBox status code.
616 * @param pVmxTransient Pointer to the VMX transient structure.
617 */
618DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
619{
620 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
621 {
622 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
623 AssertRCReturn(rc, rc);
624 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
625 }
626 return VINF_SUCCESS;
627}
628
629
630/**
631 * Reads the VM-exit interruption error code from the VMCS into the VMX
632 * transient structure.
633 *
634 * @returns VBox status code.
635 * @param pVmxTransient Pointer to the VMX transient structure.
636 */
637DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
638{
639 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
640 {
641 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
642 AssertRCReturn(rc, rc);
643 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
644 }
645 return VINF_SUCCESS;
646}
647
648
649/**
650 * Reads the VM-exit instruction length field from the VMCS into the VMX
651 * transient structure.
652 *
653 * @returns VBox status code.
654 * @param pVCpu Pointer to the VMCPU.
655 * @param pVmxTransient Pointer to the VMX transient structure.
656 */
657DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
658{
659 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
660 {
661 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
662 AssertRCReturn(rc, rc);
663 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
664 }
665 return VINF_SUCCESS;
666}
667
668
669/**
670 * Reads the VM-exit instruction-information field from the VMCS into
671 * the VMX transient structure.
672 *
673 * @returns VBox status code.
674 * @param pVmxTransient Pointer to the VMX transient structure.
675 */
676DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
677{
678 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
679 {
680 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
681 AssertRCReturn(rc, rc);
682 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
683 }
684 return VINF_SUCCESS;
685}
686
687
688/**
689 * Reads the exit code qualification from the VMCS into the VMX transient
690 * structure.
691 *
692 * @returns VBox status code.
693 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
694 * case).
695 * @param pVmxTransient Pointer to the VMX transient structure.
696 */
697DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
698{
699 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
700 {
701 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
702 AssertRCReturn(rc, rc);
703 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
704 }
705 return VINF_SUCCESS;
706}
707
708
709/**
710 * Reads the IDT-vectoring information field from the VMCS into the VMX
711 * transient structure.
712 *
713 * @returns VBox status code.
714 * @param pVmxTransient Pointer to the VMX transient structure.
715 *
716 * @remarks No-long-jump zone!!!
717 */
718DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
719{
720 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
721 {
722 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
723 AssertRCReturn(rc, rc);
724 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
725 }
726 return VINF_SUCCESS;
727}
728
729
730/**
731 * Reads the IDT-vectoring error code from the VMCS into the VMX
732 * transient structure.
733 *
734 * @returns VBox status code.
735 * @param pVmxTransient Pointer to the VMX transient structure.
736 */
737DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
738{
739 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
740 {
741 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
742 AssertRCReturn(rc, rc);
743 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
744 }
745 return VINF_SUCCESS;
746}
747
748
749/**
750 * Enters VMX root mode operation on the current CPU.
751 *
752 * @returns VBox status code.
753 * @param pVM Pointer to the VM (optional, can be NULL, after
754 * a resume).
755 * @param HCPhysCpuPage Physical address of the VMXON region.
756 * @param pvCpuPage Pointer to the VMXON region.
757 */
758static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
759{
760 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
761 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
762 Assert(pvCpuPage);
763 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
764
765 if (pVM)
766 {
767 /* Write the VMCS revision dword to the VMXON region. */
768 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
769 }
770
771 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
772 RTCCUINTREG uEflags = ASMIntDisableFlags();
773
774 /* Enable the VMX bit in CR4 if necessary. */
775 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
776
777 /* Enter VMX root mode. */
778 int rc = VMXEnable(HCPhysCpuPage);
779 if ( RT_FAILURE(rc)
780 && !(uOldCr4 & X86_CR4_VMXE))
781 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
782
783 /* Restore interrupts. */
784 ASMSetFlags(uEflags);
785 return rc;
786}
787
788
789/**
790 * Exits VMX root mode operation on the current CPU.
791 *
792 * @returns VBox status code.
793 */
794static int hmR0VmxLeaveRootMode(void)
795{
796 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
797
798 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
799 RTCCUINTREG uEflags = ASMIntDisableFlags();
800
801 /* If we're for some reason not in VMX root mode, then don't leave it. */
802 RTCCUINTREG uHostCR4 = ASMGetCR4();
803
804 int rc;
805 if (uHostCR4 & X86_CR4_VMXE)
806 {
807 /* Exit VMX root mode and clear the VMX bit in CR4. */
808 VMXDisable();
809 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
810 rc = VINF_SUCCESS;
811 }
812 else
813 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
814
815 /* Restore interrupts. */
816 ASMSetFlags(uEflags);
817 return rc;
818}
819
820
821/**
822 * Allocates and maps one physically contiguous page. The allocated page is
823 * zero'd out. (Used by various VT-x structures).
824 *
825 * @returns IPRT status code.
826 * @param pMemObj Pointer to the ring-0 memory object.
827 * @param ppVirt Where to store the virtual address of the
828 * allocation.
829 * @param pPhys Where to store the physical address of the
830 * allocation.
831 */
832DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
833{
834 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
835 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
836 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
837
838 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
839 if (RT_FAILURE(rc))
840 return rc;
841 *ppVirt = RTR0MemObjAddress(*pMemObj);
842 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
843 ASMMemZero32(*ppVirt, PAGE_SIZE);
844 return VINF_SUCCESS;
845}
846
847
848/**
849 * Frees and unmaps an allocated physical page.
850 *
851 * @param pMemObj Pointer to the ring-0 memory object.
852 * @param ppVirt Where to re-initialize the virtual address of
853 * allocation as 0.
854 * @param pHCPhys Where to re-initialize the physical address of the
855 * allocation as 0.
856 */
857DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
858{
859 AssertPtr(pMemObj);
860 AssertPtr(ppVirt);
861 AssertPtr(pHCPhys);
862 if (*pMemObj != NIL_RTR0MEMOBJ)
863 {
864 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
865 AssertRC(rc);
866 *pMemObj = NIL_RTR0MEMOBJ;
867 *ppVirt = 0;
868 *pHCPhys = 0;
869 }
870}
871
872
873/**
874 * Worker function to free VT-x related structures.
875 *
876 * @returns IPRT status code.
877 * @param pVM Pointer to the VM.
878 */
879static void hmR0VmxStructsFree(PVM pVM)
880{
881 for (VMCPUID i = 0; i < pVM->cCpus; i++)
882 {
883 PVMCPU pVCpu = &pVM->aCpus[i];
884 AssertPtr(pVCpu);
885
886 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
887 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
888
889 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
890 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
891
892 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
893 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
894 }
895
896 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
897#ifdef VBOX_WITH_CRASHDUMP_MAGIC
898 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
899#endif
900}
901
902
903/**
904 * Worker function to allocate VT-x related VM structures.
905 *
906 * @returns IPRT status code.
907 * @param pVM Pointer to the VM.
908 */
909static int hmR0VmxStructsAlloc(PVM pVM)
910{
911 /*
912 * Initialize members up-front so we can cleanup properly on allocation failure.
913 */
914#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
915 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
916 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
917 pVM->hm.s.vmx.HCPhys##a_Name = 0;
918
919#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
920 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
921 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
922 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
923
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
926#endif
927 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
928
929 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
930 for (VMCPUID i = 0; i < pVM->cCpus; i++)
931 {
932 PVMCPU pVCpu = &pVM->aCpus[i];
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
935 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
936 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
937 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
938 }
939#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
940#undef VMXLOCAL_INIT_VM_MEMOBJ
941
942 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
943 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
944 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
945 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
946
947 /*
948 * Allocate all the VT-x structures.
949 */
950 int rc = VINF_SUCCESS;
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
953 if (RT_FAILURE(rc))
954 goto cleanup;
955 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
956 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
957#endif
958
959 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
960 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
961 {
962 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
963 &pVM->hm.s.vmx.HCPhysApicAccess);
964 if (RT_FAILURE(rc))
965 goto cleanup;
966 }
967
968 /*
969 * Initialize per-VCPU VT-x structures.
970 */
971 for (VMCPUID i = 0; i < pVM->cCpus; i++)
972 {
973 PVMCPU pVCpu = &pVM->aCpus[i];
974 AssertPtr(pVCpu);
975
976 /* Allocate the VM control structure (VMCS). */
977 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
978 if (RT_FAILURE(rc))
979 goto cleanup;
980
981 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
982 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
983 {
984 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
985 &pVCpu->hm.s.vmx.HCPhysVirtApic);
986 if (RT_FAILURE(rc))
987 goto cleanup;
988 }
989
990 /*
991 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
992 * transparent accesses of specific MSRs.
993 *
994 * If the condition for enabling MSR bitmaps changes here, don't forget to
995 * update HMIsMsrBitmapsAvailable().
996 */
997 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
998 {
999 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1000 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1001 if (RT_FAILURE(rc))
1002 goto cleanup;
1003 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1004 }
1005
1006 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1007 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1008 if (RT_FAILURE(rc))
1009 goto cleanup;
1010
1011 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1012 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 return VINF_SUCCESS;
1018
1019cleanup:
1020 hmR0VmxStructsFree(pVM);
1021 return rc;
1022}
1023
1024
1025/**
1026 * Does global VT-x initialization (called during module initialization).
1027 *
1028 * @returns VBox status code.
1029 */
1030VMMR0DECL(int) VMXR0GlobalInit(void)
1031{
1032#ifdef HMVMX_USE_FUNCTION_TABLE
1033 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1034# ifdef VBOX_STRICT
1035 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1036 Assert(g_apfnVMExitHandlers[i]);
1037# endif
1038#endif
1039 return VINF_SUCCESS;
1040}
1041
1042
1043/**
1044 * Does global VT-x termination (called during module termination).
1045 */
1046VMMR0DECL(void) VMXR0GlobalTerm()
1047{
1048 /* Nothing to do currently. */
1049}
1050
1051
1052/**
1053 * Sets up and activates VT-x on the current CPU.
1054 *
1055 * @returns VBox status code.
1056 * @param pCpu Pointer to the global CPU info struct.
1057 * @param pVM Pointer to the VM (can be NULL after a host resume
1058 * operation).
1059 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1060 * fEnabledByHost is true).
1061 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1062 * @a fEnabledByHost is true).
1063 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1064 * enable VT-x on the host.
1065 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1066 */
1067VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1068 void *pvMsrs)
1069{
1070 Assert(pCpu);
1071 Assert(pvMsrs);
1072 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1073
1074 /* Enable VT-x if it's not already enabled by the host. */
1075 if (!fEnabledByHost)
1076 {
1077 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1078 if (RT_FAILURE(rc))
1079 return rc;
1080 }
1081
1082 /*
1083 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1084 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1085 */
1086 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1087 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1088 {
1089 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1090 pCpu->fFlushAsidBeforeUse = false;
1091 }
1092 else
1093 pCpu->fFlushAsidBeforeUse = true;
1094
1095 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1096 ++pCpu->cTlbFlushes;
1097
1098 return VINF_SUCCESS;
1099}
1100
1101
1102/**
1103 * Deactivates VT-x on the current CPU.
1104 *
1105 * @returns VBox status code.
1106 * @param pCpu Pointer to the global CPU info struct.
1107 * @param pvCpuPage Pointer to the VMXON region.
1108 * @param HCPhysCpuPage Physical address of the VMXON region.
1109 *
1110 * @remarks This function should never be called when SUPR0EnableVTx() or
1111 * similar was used to enable VT-x on the host.
1112 */
1113VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1114{
1115 NOREF(pCpu);
1116 NOREF(pvCpuPage);
1117 NOREF(HCPhysCpuPage);
1118
1119 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1120 return hmR0VmxLeaveRootMode();
1121}
1122
1123
1124/**
1125 * Sets the permission bits for the specified MSR in the MSR bitmap.
1126 *
1127 * @param pVCpu Pointer to the VMCPU.
1128 * @param uMSR The MSR value.
1129 * @param enmRead Whether reading this MSR causes a VM-exit.
1130 * @param enmWrite Whether writing this MSR causes a VM-exit.
1131 */
1132static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1133{
1134 int32_t iBit;
1135 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1136
1137 /*
1138 * Layout:
1139 * 0x000 - 0x3ff - Low MSR read bits
1140 * 0x400 - 0x7ff - High MSR read bits
1141 * 0x800 - 0xbff - Low MSR write bits
1142 * 0xc00 - 0xfff - High MSR write bits
1143 */
1144 if (uMsr <= 0x00001FFF)
1145 iBit = uMsr;
1146 else if ( uMsr >= 0xC0000000
1147 && uMsr <= 0xC0001FFF)
1148 {
1149 iBit = (uMsr - 0xC0000000);
1150 pbMsrBitmap += 0x400;
1151 }
1152 else
1153 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1154
1155 Assert(iBit <= 0x1fff);
1156 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1157 ASMBitSet(pbMsrBitmap, iBit);
1158 else
1159 ASMBitClear(pbMsrBitmap, iBit);
1160
1161 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1162 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1163 else
1164 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1165}
1166
1167
1168#ifdef VBOX_STRICT
1169/**
1170 * Gets the permission bits for the specified MSR in the MSR bitmap.
1171 *
1172 * @returns VBox status code.
1173 * @retval VINF_SUCCESS if the specified MSR is found.
1174 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1175 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1176 *
1177 * @param pVCpu Pointer to the VMCPU.
1178 * @param uMsr The MSR.
1179 * @param penmRead Where to store the read permissions.
1180 * @param penmWrite Where to store the write permissions.
1181 */
1182static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1183{
1184 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1185 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1186 int32_t iBit;
1187 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1188
1189 /* See hmR0VmxSetMsrPermission() for the layout. */
1190 if (uMsr <= 0x00001FFF)
1191 iBit = uMsr;
1192 else if ( uMsr >= 0xC0000000
1193 && uMsr <= 0xC0001FFF)
1194 {
1195 iBit = (uMsr - 0xC0000000);
1196 pbMsrBitmap += 0x400;
1197 }
1198 else
1199 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1200
1201 Assert(iBit <= 0x1fff);
1202 if (ASMBitTest(pbMsrBitmap, iBit))
1203 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1204 else
1205 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1206
1207 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1208 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1209 else
1210 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1211 return VINF_SUCCESS;
1212}
1213#endif /* VBOX_STRICT */
1214
1215
1216/**
1217 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1218 * area.
1219 *
1220 * @returns VBox status code.
1221 * @param pVCpu Pointer to the VMCPU.
1222 * @param cMsrs The number of MSRs.
1223 */
1224DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1225{
1226 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1227 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1228 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1229 {
1230 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1231 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1232 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1233 }
1234
1235 /* Update number of guest MSRs to load/store across the world-switch. */
1236 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1237 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1238
1239 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1240 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1241
1242 /* Update the VCPU's copy of the MSR count. */
1243 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1244
1245 return VINF_SUCCESS;
1246}
1247
1248
1249/**
1250 * Adds a new (or updates the value of an existing) guest/host MSR
1251 * pair to be swapped during the world-switch as part of the
1252 * auto-load/store MSR area in the VMCS.
1253 *
1254 * @returns true if the MSR was added -and- its value was updated, false
1255 * otherwise.
1256 * @param pVCpu Pointer to the VMCPU.
1257 * @param uMsr The MSR.
1258 * @param uGuestMsr Value of the guest MSR.
1259 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1260 * necessary.
1261 */
1262static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1263{
1264 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1265 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1266 uint32_t i;
1267 for (i = 0; i < cMsrs; i++)
1268 {
1269 if (pGuestMsr->u32Msr == uMsr)
1270 break;
1271 pGuestMsr++;
1272 }
1273
1274 bool fAdded = false;
1275 if (i == cMsrs)
1276 {
1277 ++cMsrs;
1278 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1279 AssertRC(rc);
1280
1281 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1282 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1283 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1284
1285 fAdded = true;
1286 }
1287
1288 /* Update the MSR values in the auto-load/store MSR area. */
1289 pGuestMsr->u32Msr = uMsr;
1290 pGuestMsr->u64Value = uGuestMsrValue;
1291
1292 /* Create/update the MSR slot in the host MSR area. */
1293 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1294 pHostMsr += i;
1295 pHostMsr->u32Msr = uMsr;
1296
1297 /*
1298 * Update the host MSR only when requested by the caller AND when we're
1299 * adding it to the auto-load/store area. Otherwise, it would have been
1300 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1301 */
1302 bool fUpdatedMsrValue = false;
1303 if ( fAdded
1304 && fUpdateHostMsr)
1305 {
1306 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1307 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1308 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1309 fUpdatedMsrValue = true;
1310 }
1311
1312 return fUpdatedMsrValue;
1313}
1314
1315
1316/**
1317 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1318 * auto-load/store MSR area in the VMCS.
1319 *
1320 * @returns VBox status code.
1321 * @param pVCpu Pointer to the VMCPU.
1322 * @param uMsr The MSR.
1323 */
1324static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1325{
1326 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1327 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1328 for (uint32_t i = 0; i < cMsrs; i++)
1329 {
1330 /* Find the MSR. */
1331 if (pGuestMsr->u32Msr == uMsr)
1332 {
1333 /* If it's the last MSR, simply reduce the count. */
1334 if (i == cMsrs - 1)
1335 {
1336 --cMsrs;
1337 break;
1338 }
1339
1340 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1341 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1342 pLastGuestMsr += cMsrs - 1;
1343 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1344 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1345
1346 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1347 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1348 pLastHostMsr += cMsrs - 1;
1349 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1350 pHostMsr->u64Value = pLastHostMsr->u64Value;
1351 --cMsrs;
1352 break;
1353 }
1354 pGuestMsr++;
1355 }
1356
1357 /* Update the VMCS if the count changed (meaning the MSR was found). */
1358 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1359 {
1360 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1361 AssertRCReturn(rc, rc);
1362
1363 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1364 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1365 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1366
1367 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1368 return VINF_SUCCESS;
1369 }
1370
1371 return VERR_NOT_FOUND;
1372}
1373
1374
1375/**
1376 * Checks if the specified guest MSR is part of the auto-load/store area in
1377 * the VMCS.
1378 *
1379 * @returns true if found, false otherwise.
1380 * @param pVCpu Pointer to the VMCPU.
1381 * @param uMsr The MSR to find.
1382 */
1383static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1384{
1385 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1386 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1387
1388 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1389 {
1390 if (pGuestMsr->u32Msr == uMsr)
1391 return true;
1392 }
1393 return false;
1394}
1395
1396
1397/**
1398 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1399 *
1400 * @param pVCpu Pointer to the VMCPU.
1401 *
1402 * @remarks No-long-jump zone!!!
1403 */
1404static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1405{
1406 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1407 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1408 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1409 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1410
1411 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1412 {
1413 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1414
1415 /*
1416 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1417 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1418 */
1419 if (pHostMsr->u32Msr == MSR_K6_EFER)
1420 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1421 else
1422 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1423 }
1424
1425 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1426}
1427
1428
1429#if HC_ARCH_BITS == 64
1430/**
1431 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1432 * perform lazy restoration of the host MSRs while leaving VT-x.
1433 *
1434 * @param pVCpu Pointer to the VMCPU.
1435 *
1436 * @remarks No-long-jump zone!!!
1437 */
1438static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1439{
1440 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1441
1442 /*
1443 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1444 */
1445 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1446 {
1447 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1448 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1449 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1450 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1451 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1452 }
1453}
1454
1455
1456/**
1457 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1458 * lazily while leaving VT-x.
1459 *
1460 * @returns true if it does, false otherwise.
1461 * @param pVCpu Pointer to the VMCPU.
1462 * @param uMsr The MSR to check.
1463 */
1464static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1465{
1466 NOREF(pVCpu);
1467 switch (uMsr)
1468 {
1469 case MSR_K8_LSTAR:
1470 case MSR_K6_STAR:
1471 case MSR_K8_SF_MASK:
1472 case MSR_K8_KERNEL_GS_BASE:
1473 return true;
1474 }
1475 return false;
1476}
1477
1478
1479/**
1480 * Saves a set of guest MSRs back into the guest-CPU context.
1481 *
1482 * @param pVCpu Pointer to the VMCPU.
1483 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1484 * out-of-sync. Make sure to update the required fields
1485 * before using them.
1486 *
1487 * @remarks No-long-jump zone!!!
1488 */
1489static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1490{
1491 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1492 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1493
1494 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1495 {
1496 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1497 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1498 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1499 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1500 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1501 }
1502}
1503
1504
1505/**
1506 * Loads a set of guests MSRs to allow read/passthru to the guest.
1507 *
1508 * The name of this function is slightly confusing. This function does NOT
1509 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1510 * common prefix for functions dealing with "lazy restoration" of the shared
1511 * MSRs.
1512 *
1513 * @param pVCpu Pointer to the VMCPU.
1514 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1515 * out-of-sync. Make sure to update the required fields
1516 * before using them.
1517 *
1518 * @remarks No-long-jump zone!!!
1519 */
1520static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1521{
1522 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1523 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1524
1525#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1526 do { \
1527 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1528 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1529 else \
1530 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1531 } while (0)
1532
1533 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1534 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1535 {
1536 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1537 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1538 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1539 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1540 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1541 }
1542 else
1543 {
1544 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1545 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1546 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1547 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1548 }
1549
1550#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1551}
1552
1553
1554/**
1555 * Performs lazy restoration of the set of host MSRs if they were previously
1556 * loaded with guest MSR values.
1557 *
1558 * @param pVCpu Pointer to the VMCPU.
1559 *
1560 * @remarks No-long-jump zone!!!
1561 * @remarks The guest MSRs should have been saved back into the guest-CPU
1562 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1563 */
1564static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1565{
1566 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1567 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1568
1569 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1570 {
1571 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1572 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1573 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1574 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1575 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1576 }
1577 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1578}
1579#endif /* HC_ARCH_BITS == 64 */
1580
1581
1582/**
1583 * Verifies that our cached values of the VMCS controls are all
1584 * consistent with what's actually present in the VMCS.
1585 *
1586 * @returns VBox status code.
1587 * @param pVCpu Pointer to the VMCPU.
1588 */
1589static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1590{
1591 uint32_t u32Val;
1592 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1593 AssertRCReturn(rc, rc);
1594 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1595 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1596
1597 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1598 AssertRCReturn(rc, rc);
1599 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1600 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1601
1602 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1603 AssertRCReturn(rc, rc);
1604 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1605 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1606
1607 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1608 AssertRCReturn(rc, rc);
1609 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1610 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1611
1612 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1613 {
1614 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1615 AssertRCReturn(rc, rc);
1616 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1617 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1618 }
1619
1620 return VINF_SUCCESS;
1621}
1622
1623
1624#ifdef VBOX_STRICT
1625/**
1626 * Verifies that our cached host EFER value has not changed
1627 * since we cached it.
1628 *
1629 * @param pVCpu Pointer to the VMCPU.
1630 */
1631static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1632{
1633 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1634
1635 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1636 {
1637 uint64_t u64Val;
1638 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1639 AssertRC(rc);
1640
1641 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1642 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1643 }
1644}
1645
1646
1647/**
1648 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1649 * VMCS are correct.
1650 *
1651 * @param pVCpu Pointer to the VMCPU.
1652 */
1653static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1654{
1655 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1656
1657 /* Verify MSR counts in the VMCS are what we think it should be. */
1658 uint32_t cMsrs;
1659 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1660 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1661
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1663 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1664
1665 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1666 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1667
1668 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1669 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1670 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1671 {
1672 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1673 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1674 pGuestMsr->u32Msr, cMsrs));
1675
1676 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1677 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1678 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1679
1680 /* Verify that the permissions are as expected in the MSR bitmap. */
1681 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1682 {
1683 VMXMSREXITREAD enmRead;
1684 VMXMSREXITWRITE enmWrite;
1685 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1686 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1687 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1688 {
1689 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1690 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1691 }
1692 else
1693 {
1694 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1695 pGuestMsr->u32Msr, cMsrs));
1696 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1697 pGuestMsr->u32Msr, cMsrs));
1698 }
1699 }
1700 }
1701}
1702#endif /* VBOX_STRICT */
1703
1704
1705/**
1706 * Flushes the TLB using EPT.
1707 *
1708 * @returns VBox status code.
1709 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1710 * enmFlush).
1711 * @param enmFlush Type of flush.
1712 *
1713 * @remarks Caller is responsible for making sure this function is called only
1714 * when NestedPaging is supported and providing @a enmFlush that is
1715 * supported by the CPU.
1716 * @remarks Can be called with interrupts disabled.
1717 */
1718static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1719{
1720 uint64_t au64Descriptor[2];
1721 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1722 au64Descriptor[0] = 0;
1723 else
1724 {
1725 Assert(pVCpu);
1726 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1727 }
1728 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1729
1730 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1731 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1732 rc));
1733 if ( RT_SUCCESS(rc)
1734 && pVCpu)
1735 {
1736 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1737 }
1738}
1739
1740
1741/**
1742 * Flushes the TLB using VPID.
1743 *
1744 * @returns VBox status code.
1745 * @param pVM Pointer to the VM.
1746 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1747 * enmFlush).
1748 * @param enmFlush Type of flush.
1749 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1750 * on @a enmFlush).
1751 *
1752 * @remarks Can be called with interrupts disabled.
1753 */
1754static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1755{
1756 NOREF(pVM);
1757 AssertPtr(pVM);
1758 Assert(pVM->hm.s.vmx.fVpid);
1759
1760 uint64_t au64Descriptor[2];
1761 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1762 {
1763 au64Descriptor[0] = 0;
1764 au64Descriptor[1] = 0;
1765 }
1766 else
1767 {
1768 AssertPtr(pVCpu);
1769 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1770 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1771 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1772 au64Descriptor[1] = GCPtr;
1773 }
1774
1775 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1776 AssertMsg(rc == VINF_SUCCESS,
1777 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1778 if ( RT_SUCCESS(rc)
1779 && pVCpu)
1780 {
1781 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1782 }
1783}
1784
1785
1786/**
1787 * Invalidates a guest page by guest virtual address. Only relevant for
1788 * EPT/VPID, otherwise there is nothing really to invalidate.
1789 *
1790 * @returns VBox status code.
1791 * @param pVM Pointer to the VM.
1792 * @param pVCpu Pointer to the VMCPU.
1793 * @param GCVirt Guest virtual address of the page to invalidate.
1794 */
1795VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1796{
1797 AssertPtr(pVM);
1798 AssertPtr(pVCpu);
1799 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1800
1801 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1802 if (!fFlushPending)
1803 {
1804 /*
1805 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1806 * See @bugref{6043} and @bugref{6177}.
1807 *
1808 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1809 * function maybe called in a loop with individual addresses.
1810 */
1811 if (pVM->hm.s.vmx.fVpid)
1812 {
1813 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1814 {
1815 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1816 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1817 }
1818 else
1819 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1820 }
1821 else if (pVM->hm.s.fNestedPaging)
1822 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1823 }
1824
1825 return VINF_SUCCESS;
1826}
1827
1828
1829/**
1830 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1831 * otherwise there is nothing really to invalidate.
1832 *
1833 * @returns VBox status code.
1834 * @param pVM Pointer to the VM.
1835 * @param pVCpu Pointer to the VMCPU.
1836 * @param GCPhys Guest physical address of the page to invalidate.
1837 */
1838VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1839{
1840 NOREF(pVM); NOREF(GCPhys);
1841 LogFlowFunc(("%RGp\n", GCPhys));
1842
1843 /*
1844 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1845 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1846 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1847 */
1848 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1849 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1850 return VINF_SUCCESS;
1851}
1852
1853
1854/**
1855 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1856 * case where neither EPT nor VPID is supported by the CPU.
1857 *
1858 * @param pVM Pointer to the VM.
1859 * @param pVCpu Pointer to the VMCPU.
1860 * @param pCpu Pointer to the global HM struct.
1861 *
1862 * @remarks Called with interrupts disabled.
1863 */
1864static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1865{
1866 AssertPtr(pVCpu);
1867 AssertPtr(pCpu);
1868 NOREF(pVM);
1869
1870 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1871
1872 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1873#if 0
1874 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1875 pVCpu->hm.s.TlbShootdown.cPages = 0;
1876#endif
1877
1878 Assert(pCpu->idCpu != NIL_RTCPUID);
1879 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1880 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1881 pVCpu->hm.s.fForceTLBFlush = false;
1882 return;
1883}
1884
1885
1886/**
1887 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1888 *
1889 * @param pVM Pointer to the VM.
1890 * @param pVCpu Pointer to the VMCPU.
1891 * @param pCpu Pointer to the global HM CPU struct.
1892 * @remarks All references to "ASID" in this function pertains to "VPID" in
1893 * Intel's nomenclature. The reason is, to avoid confusion in compare
1894 * statements since the host-CPU copies are named "ASID".
1895 *
1896 * @remarks Called with interrupts disabled.
1897 */
1898static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1899{
1900#ifdef VBOX_WITH_STATISTICS
1901 bool fTlbFlushed = false;
1902# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1903# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1904 if (!fTlbFlushed) \
1905 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1906 } while (0)
1907#else
1908# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1909# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1910#endif
1911
1912 AssertPtr(pVM);
1913 AssertPtr(pCpu);
1914 AssertPtr(pVCpu);
1915 Assert(pCpu->idCpu != NIL_RTCPUID);
1916
1917 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1918 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1919 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1920
1921 /*
1922 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1923 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1924 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1925 */
1926 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1927 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1928 {
1929 ++pCpu->uCurrentAsid;
1930 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1931 {
1932 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1933 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1934 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1935 }
1936
1937 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1938 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1939 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1940
1941 /*
1942 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1943 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1944 */
1945 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1946 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1947 HMVMX_SET_TAGGED_TLB_FLUSHED();
1948 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1949 }
1950
1951 /* Check for explicit TLB shootdowns. */
1952 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1953 {
1954 /*
1955 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1956 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1957 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1958 * but not guest-physical mappings.
1959 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1960 */
1961 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1962 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1963 HMVMX_SET_TAGGED_TLB_FLUSHED();
1964 }
1965
1966 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1967 * where it is commented out. Support individual entry flushing
1968 * someday. */
1969#if 0
1970 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1971 {
1972 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1973
1974 /*
1975 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1976 * as supported by the CPU.
1977 */
1978 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1979 {
1980 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1981 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1982 }
1983 else
1984 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1985
1986 HMVMX_SET_TAGGED_TLB_FLUSHED();
1987 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1988 pVCpu->hm.s.TlbShootdown.cPages = 0;
1989 }
1990#endif
1991
1992 pVCpu->hm.s.fForceTLBFlush = false;
1993
1994 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1995
1996 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1997 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1998 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1999 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2000 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2001 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2002 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2003 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2004 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2005
2006 /* Update VMCS with the VPID. */
2007 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2008 AssertRC(rc);
2009
2010#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2011}
2012
2013
2014/**
2015 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2016 *
2017 * @returns VBox status code.
2018 * @param pVM Pointer to the VM.
2019 * @param pVCpu Pointer to the VMCPU.
2020 * @param pCpu Pointer to the global HM CPU struct.
2021 *
2022 * @remarks Called with interrupts disabled.
2023 */
2024static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2025{
2026 AssertPtr(pVM);
2027 AssertPtr(pVCpu);
2028 AssertPtr(pCpu);
2029 Assert(pCpu->idCpu != NIL_RTCPUID);
2030 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2031 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2032
2033 /*
2034 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2035 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2036 */
2037 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2038 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2039 {
2040 pVCpu->hm.s.fForceTLBFlush = true;
2041 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2042 }
2043
2044 /* Check for explicit TLB shootdown flushes. */
2045 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2046 {
2047 pVCpu->hm.s.fForceTLBFlush = true;
2048 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2049 }
2050
2051 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2052 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2053
2054 if (pVCpu->hm.s.fForceTLBFlush)
2055 {
2056 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2057 pVCpu->hm.s.fForceTLBFlush = false;
2058 }
2059 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2060 * where it is commented out. Support individual entry flushing
2061 * someday. */
2062#if 0
2063 else
2064 {
2065 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2066 {
2067 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2068 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2069 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2070 }
2071 else
2072 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2073
2074 pVCpu->hm.s.TlbShootdown.cPages = 0;
2075 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2076 }
2077#endif
2078}
2079
2080
2081/**
2082 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2083 *
2084 * @returns VBox status code.
2085 * @param pVM Pointer to the VM.
2086 * @param pVCpu Pointer to the VMCPU.
2087 * @param pCpu Pointer to the global HM CPU struct.
2088 *
2089 * @remarks Called with interrupts disabled.
2090 */
2091static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2092{
2093 AssertPtr(pVM);
2094 AssertPtr(pVCpu);
2095 AssertPtr(pCpu);
2096 Assert(pCpu->idCpu != NIL_RTCPUID);
2097 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2098 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2099
2100 /*
2101 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2102 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2103 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2104 */
2105 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2106 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2107 {
2108 pVCpu->hm.s.fForceTLBFlush = true;
2109 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2110 }
2111
2112 /* Check for explicit TLB shootdown flushes. */
2113 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2114 {
2115 /*
2116 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2117 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2118 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2119 */
2120 pVCpu->hm.s.fForceTLBFlush = true;
2121 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2122 }
2123
2124 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2125 if (pVCpu->hm.s.fForceTLBFlush)
2126 {
2127 ++pCpu->uCurrentAsid;
2128 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2129 {
2130 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2131 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2132 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2133 }
2134
2135 pVCpu->hm.s.fForceTLBFlush = false;
2136 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2137 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2138 if (pCpu->fFlushAsidBeforeUse)
2139 {
2140 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2142 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2143 {
2144 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2145 pCpu->fFlushAsidBeforeUse = false;
2146 }
2147 else
2148 {
2149 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2150 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2151 }
2152 }
2153 }
2154 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2155 * where it is commented out. Support individual entry flushing
2156 * someday. */
2157#if 0
2158 else
2159 {
2160 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2161 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2162 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2163 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2164
2165 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2166 {
2167 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2168 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2169 {
2170 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2171 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2172 }
2173 else
2174 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2175
2176 pVCpu->hm.s.TlbShootdown.cPages = 0;
2177 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2178 }
2179 else
2180 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2181 }
2182#endif
2183
2184 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2185 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2186 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2187 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2188 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2189 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2190 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2191
2192 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2193 AssertRC(rc);
2194}
2195
2196
2197/**
2198 * Flushes the guest TLB entry based on CPU capabilities.
2199 *
2200 * @param pVCpu Pointer to the VMCPU.
2201 * @param pCpu Pointer to the global HM CPU struct.
2202 */
2203DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2204{
2205#ifdef HMVMX_ALWAYS_FLUSH_TLB
2206 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2207#endif
2208 PVM pVM = pVCpu->CTX_SUFF(pVM);
2209 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2210 {
2211 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2212 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2213 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2214 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2215 default:
2216 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2217 break;
2218 }
2219
2220 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2221 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2222
2223 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2224}
2225
2226
2227/**
2228 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2229 * TLB entries from the host TLB before VM-entry.
2230 *
2231 * @returns VBox status code.
2232 * @param pVM Pointer to the VM.
2233 */
2234static int hmR0VmxSetupTaggedTlb(PVM pVM)
2235{
2236 /*
2237 * Determine optimal flush type for Nested Paging.
2238 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2239 * guest execution (see hmR3InitFinalizeR0()).
2240 */
2241 if (pVM->hm.s.fNestedPaging)
2242 {
2243 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2244 {
2245 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2246 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2247 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2248 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2249 else
2250 {
2251 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2252 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2253 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2254 }
2255
2256 /* Make sure the write-back cacheable memory type for EPT is supported. */
2257 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2258 {
2259 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2260 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2261 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2262 }
2263 }
2264 else
2265 {
2266 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2267 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2268 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2269 }
2270 }
2271
2272 /*
2273 * Determine optimal flush type for VPID.
2274 */
2275 if (pVM->hm.s.vmx.fVpid)
2276 {
2277 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2278 {
2279 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2280 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2281 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2282 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2283 else
2284 {
2285 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2286 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2287 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2288 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2289 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2290 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2291 pVM->hm.s.vmx.fVpid = false;
2292 }
2293 }
2294 else
2295 {
2296 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2297 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2298 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2299 pVM->hm.s.vmx.fVpid = false;
2300 }
2301 }
2302
2303 /*
2304 * Setup the handler for flushing tagged-TLBs.
2305 */
2306 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2307 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2308 else if (pVM->hm.s.fNestedPaging)
2309 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2310 else if (pVM->hm.s.vmx.fVpid)
2311 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2312 else
2313 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2314 return VINF_SUCCESS;
2315}
2316
2317
2318/**
2319 * Sets up pin-based VM-execution controls in the VMCS.
2320 *
2321 * @returns VBox status code.
2322 * @param pVM Pointer to the VM.
2323 * @param pVCpu Pointer to the VMCPU.
2324 */
2325static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2326{
2327 AssertPtr(pVM);
2328 AssertPtr(pVCpu);
2329
2330 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2331 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2332
2333 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2334 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2335
2336 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2337 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2338
2339 /* Enable the VMX preemption timer. */
2340 if (pVM->hm.s.vmx.fUsePreemptTimer)
2341 {
2342 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2343 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2344 }
2345
2346 if ((val & zap) != val)
2347 {
2348 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2349 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2350 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2351 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2352 }
2353
2354 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2355 AssertRCReturn(rc, rc);
2356
2357 pVCpu->hm.s.vmx.u32PinCtls = val;
2358 return rc;
2359}
2360
2361
2362/**
2363 * Sets up processor-based VM-execution controls in the VMCS.
2364 *
2365 * @returns VBox status code.
2366 * @param pVM Pointer to the VM.
2367 * @param pVMCPU Pointer to the VMCPU.
2368 */
2369static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2370{
2371 AssertPtr(pVM);
2372 AssertPtr(pVCpu);
2373
2374 int rc = VERR_INTERNAL_ERROR_5;
2375 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2376 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2377
2378 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2379 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2380 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2381 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2382 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2383 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2384 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2385
2386 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2387 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2388 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2389 {
2390 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2391 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2392 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2393 }
2394
2395 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2396 if (!pVM->hm.s.fNestedPaging)
2397 {
2398 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2399 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2400 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2401 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2402 }
2403
2404 /* Use TPR shadowing if supported by the CPU. */
2405 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2406 {
2407 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2408 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2409 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2410 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2411 AssertRCReturn(rc, rc);
2412
2413 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2414 /* CR8 writes cause a VM-exit based on TPR threshold. */
2415 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2416 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2417 }
2418 else
2419 {
2420 /*
2421 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2422 * Set this control only for 64-bit guests.
2423 */
2424 if (pVM->hm.s.fAllow64BitGuests)
2425 {
2426 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2427 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2428 }
2429 }
2430
2431 /* Use MSR-bitmaps if supported by the CPU. */
2432 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2433 {
2434 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2435
2436 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2437 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2438 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2439 AssertRCReturn(rc, rc);
2440
2441 /*
2442 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2443 * automatically using dedicated fields in the VMCS.
2444 */
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2449 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2450
2451#if HC_ARCH_BITS == 64
2452 /*
2453 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2454 */
2455 if (pVM->hm.s.fAllow64BitGuests)
2456 {
2457 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2458 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2459 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2460 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2461 }
2462#endif
2463 }
2464
2465 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2466 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2467 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2468
2469 if ((val & zap) != val)
2470 {
2471 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2472 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2473 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2474 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2475 }
2476
2477 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2478 AssertRCReturn(rc, rc);
2479
2480 pVCpu->hm.s.vmx.u32ProcCtls = val;
2481
2482 /*
2483 * Secondary processor-based VM-execution controls.
2484 */
2485 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2486 {
2487 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2488 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2489
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2492
2493 if (pVM->hm.s.fNestedPaging)
2494 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2495 else
2496 {
2497 /*
2498 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2499 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2500 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2501 */
2502 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2503 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2504 }
2505
2506 if (pVM->hm.s.vmx.fVpid)
2507 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2508
2509 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2510 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2511
2512 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2513 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2514 * done dynamically. */
2515 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2516 {
2517 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2518 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2519 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2520 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2521 AssertRCReturn(rc, rc);
2522 }
2523
2524 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2525 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2526
2527 if ((val & zap) != val)
2528 {
2529 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2530 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2531 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2532 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2533 }
2534
2535 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2536 AssertRCReturn(rc, rc);
2537
2538 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2539 }
2540 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2541 {
2542 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2543 "available\n"));
2544 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2545 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2546 }
2547
2548 return VINF_SUCCESS;
2549}
2550
2551
2552/**
2553 * Sets up miscellaneous (everything other than Pin & Processor-based
2554 * VM-execution) control fields in the VMCS.
2555 *
2556 * @returns VBox status code.
2557 * @param pVM Pointer to the VM.
2558 * @param pVCpu Pointer to the VMCPU.
2559 */
2560static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2561{
2562 NOREF(pVM);
2563 AssertPtr(pVM);
2564 AssertPtr(pVCpu);
2565
2566 int rc = VERR_GENERAL_FAILURE;
2567
2568 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2569#if 0
2570 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2571 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2572 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2573
2574 /*
2575 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2576 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2577 * We thus use the exception bitmap to control it rather than use both.
2578 */
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2580 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2581
2582 /** @todo Explore possibility of using IO-bitmaps. */
2583 /* All IO & IOIO instructions cause VM-exits. */
2584 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2585 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2586
2587 /* Initialize the MSR-bitmap area. */
2588 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2590 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2591#endif
2592
2593 /* Setup MSR auto-load/store area. */
2594 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2595 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2596 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2597 AssertRCReturn(rc, rc);
2598 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2599 AssertRCReturn(rc, rc);
2600
2601 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2602 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2603 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2604 AssertRCReturn(rc, rc);
2605
2606 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2607 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2608 AssertRCReturn(rc, rc);
2609
2610 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2611#if 0
2612 /* Setup debug controls */
2613 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2614 AssertRCReturn(rc, rc);
2615 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2616 AssertRCReturn(rc, rc);
2617#endif
2618
2619 return rc;
2620}
2621
2622
2623/**
2624 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2625 *
2626 * @returns VBox status code.
2627 * @param pVM Pointer to the VM.
2628 * @param pVCpu Pointer to the VMCPU.
2629 */
2630static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2631{
2632 AssertPtr(pVM);
2633 AssertPtr(pVCpu);
2634
2635 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2636
2637 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2638
2639 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2640 if (!pVM->hm.s.fNestedPaging)
2641 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2642
2643 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2644 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2645 AssertRCReturn(rc, rc);
2646 return rc;
2647}
2648
2649
2650/**
2651 * Sets up the initial guest-state mask. The guest-state mask is consulted
2652 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2653 * for the nested virtualization case (as it would cause a VM-exit).
2654 *
2655 * @param pVCpu Pointer to the VMCPU.
2656 */
2657static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2658{
2659 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2660 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2661 return VINF_SUCCESS;
2662}
2663
2664
2665/**
2666 * Does per-VM VT-x initialization.
2667 *
2668 * @returns VBox status code.
2669 * @param pVM Pointer to the VM.
2670 */
2671VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2672{
2673 LogFlowFunc(("pVM=%p\n", pVM));
2674
2675 int rc = hmR0VmxStructsAlloc(pVM);
2676 if (RT_FAILURE(rc))
2677 {
2678 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2679 return rc;
2680 }
2681
2682 return VINF_SUCCESS;
2683}
2684
2685
2686/**
2687 * Does per-VM VT-x termination.
2688 *
2689 * @returns VBox status code.
2690 * @param pVM Pointer to the VM.
2691 */
2692VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2693{
2694 LogFlowFunc(("pVM=%p\n", pVM));
2695
2696#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2697 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2698 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2699#endif
2700 hmR0VmxStructsFree(pVM);
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Sets up the VM for execution under VT-x.
2707 * This function is only called once per-VM during initialization.
2708 *
2709 * @returns VBox status code.
2710 * @param pVM Pointer to the VM.
2711 */
2712VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2713{
2714 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2715 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2716
2717 LogFlowFunc(("pVM=%p\n", pVM));
2718
2719 /*
2720 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2721 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2722 */
2723 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2724 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2725 || !pVM->hm.s.vmx.pRealModeTSS))
2726 {
2727 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2728 return VERR_INTERNAL_ERROR;
2729 }
2730
2731#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2732 /*
2733 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2734 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2735 */
2736 if ( pVM->hm.s.fAllow64BitGuests
2737 && !HMVMX_IS_64BIT_HOST_MODE())
2738 {
2739 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2740 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2741 }
2742#endif
2743
2744 /* Initialize these always, see hmR3InitFinalizeR0().*/
2745 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2746 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2747
2748 /* Setup the tagged-TLB flush handlers. */
2749 int rc = hmR0VmxSetupTaggedTlb(pVM);
2750 if (RT_FAILURE(rc))
2751 {
2752 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2753 return rc;
2754 }
2755
2756 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2757 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2758#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2759 if ( HMVMX_IS_64BIT_HOST_MODE()
2760 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2761 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2762 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2763 {
2764 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2765 }
2766#endif
2767
2768 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2769 {
2770 PVMCPU pVCpu = &pVM->aCpus[i];
2771 AssertPtr(pVCpu);
2772 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2773
2774 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2775 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2776
2777 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2778 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2779 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2780
2781 /* Set revision dword at the beginning of the VMCS structure. */
2782 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2783
2784 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2785 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2787 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2788
2789 /* Load this VMCS as the current VMCS. */
2790 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2791 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2792 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2793
2794 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2796 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2797
2798 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2800 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2801
2802 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2804 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2805
2806 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2815 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2816 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2817 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2818#endif
2819
2820 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2821 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2822 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2823 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2824
2825 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2826
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2828 }
2829
2830 return VINF_SUCCESS;
2831}
2832
2833
2834/**
2835 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2836 * the VMCS.
2837 *
2838 * @returns VBox status code.
2839 * @param pVM Pointer to the VM.
2840 * @param pVCpu Pointer to the VMCPU.
2841 */
2842DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2843{
2844 NOREF(pVM); NOREF(pVCpu);
2845
2846 RTCCUINTREG uReg = ASMGetCR0();
2847 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2848 AssertRCReturn(rc, rc);
2849
2850#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2851 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2852 if (HMVMX_IS_64BIT_HOST_MODE())
2853 {
2854 uint64_t uRegCR3 = HMR0Get64bitCR3();
2855 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2856 }
2857 else
2858#endif
2859 {
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 }
2863 AssertRCReturn(rc, rc);
2864
2865 uReg = ASMGetCR4();
2866 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2867 AssertRCReturn(rc, rc);
2868 return rc;
2869}
2870
2871
2872#if HC_ARCH_BITS == 64
2873/**
2874 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2875 * requirements. See hmR0VmxSaveHostSegmentRegs().
2876 */
2877# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2878 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2879 { \
2880 bool fValidSelector = true; \
2881 if ((selValue) & X86_SEL_LDT) \
2882 { \
2883 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2884 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2885 } \
2886 if (fValidSelector) \
2887 { \
2888 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2889 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2890 } \
2891 (selValue) = 0; \
2892 }
2893#endif
2894
2895
2896/**
2897 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2898 * the host-state area in the VMCS.
2899 *
2900 * @returns VBox status code.
2901 * @param pVM Pointer to the VM.
2902 * @param pVCpu Pointer to the VMCPU.
2903 */
2904DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2905{
2906 int rc = VERR_INTERNAL_ERROR_5;
2907
2908#if HC_ARCH_BITS == 64
2909 /*
2910 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2911 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2912 */
2913 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2914 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2915#endif
2916
2917 /*
2918 * Host DS, ES, FS and GS segment registers.
2919 */
2920#if HC_ARCH_BITS == 64
2921 RTSEL uSelDS = ASMGetDS();
2922 RTSEL uSelES = ASMGetES();
2923 RTSEL uSelFS = ASMGetFS();
2924 RTSEL uSelGS = ASMGetGS();
2925#else
2926 RTSEL uSelDS = 0;
2927 RTSEL uSelES = 0;
2928 RTSEL uSelFS = 0;
2929 RTSEL uSelGS = 0;
2930#endif
2931
2932 /* Recalculate which host-state bits need to be manually restored. */
2933 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2934
2935 /*
2936 * Host CS and SS segment registers.
2937 */
2938#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2939 RTSEL uSelCS;
2940 RTSEL uSelSS;
2941 if (HMVMX_IS_64BIT_HOST_MODE())
2942 {
2943 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2944 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2945 }
2946 else
2947 {
2948 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2949 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2950 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2951 }
2952#else
2953 RTSEL uSelCS = ASMGetCS();
2954 RTSEL uSelSS = ASMGetSS();
2955#endif
2956
2957 /*
2958 * Host TR segment register.
2959 */
2960 RTSEL uSelTR = ASMGetTR();
2961
2962#if HC_ARCH_BITS == 64
2963 /*
2964 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2965 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2966 */
2967 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2968 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2969 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2970 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2971# undef VMXLOCAL_ADJUST_HOST_SEG
2972#endif
2973
2974 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2975 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2976 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2977 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2978 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2979 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2980 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2981 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2982 Assert(uSelCS);
2983 Assert(uSelTR);
2984
2985 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2986#if 0
2987 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2988 Assert(uSelSS != 0);
2989#endif
2990
2991 /* Write these host selector fields into the host-state area in the VMCS. */
2992 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2994#if HC_ARCH_BITS == 64
2995 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2999#endif
3000 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3001
3002 /*
3003 * Host GDTR and IDTR.
3004 */
3005 RTGDTR Gdtr;
3006 RT_ZERO(Gdtr);
3007#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3008 if (HMVMX_IS_64BIT_HOST_MODE())
3009 {
3010 X86XDTR64 Gdtr64;
3011 X86XDTR64 Idtr64;
3012 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3013 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3014 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3015
3016 Gdtr.cbGdt = Gdtr64.cb;
3017 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3018 }
3019 else
3020#endif
3021 {
3022 RTIDTR Idtr;
3023 ASMGetGDTR(&Gdtr);
3024 ASMGetIDTR(&Idtr);
3025 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3026 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3027
3028#if HC_ARCH_BITS == 64
3029 /*
3030 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3031 * maximum limit (0xffff) on every VM-exit.
3032 */
3033 if (Gdtr.cbGdt != 0xffff)
3034 {
3035 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3036 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3037 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3038 }
3039
3040 /*
3041 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3042 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3043 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3044 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3045 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3046 * hosts where we are pretty sure it won't cause trouble.
3047 */
3048# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3049 if (Idtr.cbIdt < 0x0fff)
3050# else
3051 if (Idtr.cbIdt != 0xffff)
3052# endif
3053 {
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3055 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3056 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3057 }
3058#endif
3059 }
3060
3061 /*
3062 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3063 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3064 */
3065 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3066 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3067 VERR_VMX_INVALID_HOST_STATE);
3068
3069 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3070#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3071 if (HMVMX_IS_64BIT_HOST_MODE())
3072 {
3073 /* We need the 64-bit TR base for hybrid darwin. */
3074 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3075 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3076 }
3077 else
3078#endif
3079 {
3080 uintptr_t uTRBase;
3081#if HC_ARCH_BITS == 64
3082 uTRBase = X86DESC64_BASE(pDesc);
3083
3084 /*
3085 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3086 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3087 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3088 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3089 *
3090 * [1] See Intel spec. 3.5 "System Descriptor Types".
3091 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3092 */
3093 Assert(pDesc->System.u4Type == 11);
3094 if ( pDesc->System.u16LimitLow != 0x67
3095 || pDesc->System.u4LimitHigh)
3096 {
3097 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3098 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3099 if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3100 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3101 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3102
3103 /* Store the GDTR here as we need it while restoring TR. */
3104 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3105 }
3106#else
3107 uTRBase = X86DESC_BASE(pDesc);
3108#endif
3109 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3110 }
3111 AssertRCReturn(rc, rc);
3112
3113 /*
3114 * Host FS base and GS base.
3115 */
3116#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3117 if (HMVMX_IS_64BIT_HOST_MODE())
3118 {
3119 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3120 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3121 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3122 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3123
3124# if HC_ARCH_BITS == 64
3125 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3126 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3127 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3128 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3129 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3130# endif
3131 }
3132#endif
3133 return rc;
3134}
3135
3136
3137/**
3138 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3139 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3140 * the host after every successful VM-exit.
3141 *
3142 * @returns VBox status code.
3143 * @param pVM Pointer to the VM.
3144 * @param pVCpu Pointer to the VMCPU.
3145 *
3146 * @remarks No-long-jump zone!!!
3147 */
3148DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3149{
3150 NOREF(pVM);
3151
3152 AssertPtr(pVCpu);
3153 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3154
3155 int rc = VINF_SUCCESS;
3156#if HC_ARCH_BITS == 64
3157 if (pVM->hm.s.fAllow64BitGuests)
3158 hmR0VmxLazySaveHostMsrs(pVCpu);
3159#endif
3160
3161 /*
3162 * Host Sysenter MSRs.
3163 */
3164 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3165 AssertRCReturn(rc, rc);
3166#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3167 if (HMVMX_IS_64BIT_HOST_MODE())
3168 {
3169 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3170 AssertRCReturn(rc, rc);
3171 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3172 }
3173 else
3174 {
3175 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3176 AssertRCReturn(rc, rc);
3177 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3178 }
3179#elif HC_ARCH_BITS == 32
3180 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3181 AssertRCReturn(rc, rc);
3182 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3183#else
3184 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3185 AssertRCReturn(rc, rc);
3186 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3187#endif
3188 AssertRCReturn(rc, rc);
3189
3190 /*
3191 * Host EFER MSR.
3192 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3193 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3194 */
3195 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3196 {
3197 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3198 AssertRCReturn(rc, rc);
3199 }
3200
3201 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3202 * hmR0VmxLoadGuestExitCtls() !! */
3203
3204 return rc;
3205}
3206
3207
3208/**
3209 * Figures out if we need to swap the EFER MSR which is
3210 * particularly expensive.
3211 *
3212 * We check all relevant bits. For now, that's everything
3213 * besides LMA/LME, as these two bits are handled by VM-entry,
3214 * see hmR0VmxLoadGuestExitCtls() and
3215 * hmR0VMxLoadGuestEntryCtls().
3216 *
3217 * @returns true if we need to load guest EFER, false otherwise.
3218 * @param pVCpu Pointer to the VMCPU.
3219 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3220 * out-of-sync. Make sure to update the required fields
3221 * before using them.
3222 *
3223 * @remarks Requires EFER, CR4.
3224 * @remarks No-long-jump zone!!!
3225 */
3226static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3227{
3228#ifdef HMVMX_ALWAYS_SWAP_EFER
3229 return true;
3230#endif
3231
3232#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3233 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3234 if (CPUMIsGuestInLongMode(pVCpu))
3235 return false;
3236#endif
3237
3238 PVM pVM = pVCpu->CTX_SUFF(pVM);
3239 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3240 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3241
3242 /*
3243 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3244 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3245 */
3246 if ( CPUMIsGuestInLongMode(pVCpu)
3247 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3248 {
3249 return true;
3250 }
3251
3252 /*
3253 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3254 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3255 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3256 */
3257 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3258 && (pMixedCtx->cr0 & X86_CR0_PG)
3259 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3260 {
3261 /* Assert that host is PAE capable. */
3262 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3263 return true;
3264 }
3265
3266 /** @todo Check the latest Intel spec. for any other bits,
3267 * like SMEP/SMAP? */
3268 return false;
3269}
3270
3271
3272/**
3273 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3274 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3275 * controls".
3276 *
3277 * @returns VBox status code.
3278 * @param pVCpu Pointer to the VMCPU.
3279 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3280 * out-of-sync. Make sure to update the required fields
3281 * before using them.
3282 *
3283 * @remarks Requires EFER.
3284 * @remarks No-long-jump zone!!!
3285 */
3286DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3287{
3288 int rc = VINF_SUCCESS;
3289 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3290 {
3291 PVM pVM = pVCpu->CTX_SUFF(pVM);
3292 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3293 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3294
3295 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3296 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3297
3298 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3299 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3300 {
3301 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3302 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3303 }
3304 else
3305 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3306
3307 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3308 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3309 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3310 {
3311 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3312 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3313 }
3314
3315 /*
3316 * The following should -not- be set (since we're not in SMM mode):
3317 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3318 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3319 */
3320
3321 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3322 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3323
3324 if ((val & zap) != val)
3325 {
3326 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3327 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3328 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3329 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3330 }
3331
3332 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3333 AssertRCReturn(rc, rc);
3334
3335 pVCpu->hm.s.vmx.u32EntryCtls = val;
3336 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3337 }
3338 return rc;
3339}
3340
3341
3342/**
3343 * Sets up the VM-exit controls in the VMCS.
3344 *
3345 * @returns VBox status code.
3346 * @param pVM Pointer to the VM.
3347 * @param pVCpu Pointer to the VMCPU.
3348 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3349 * out-of-sync. Make sure to update the required fields
3350 * before using them.
3351 *
3352 * @remarks Requires EFER.
3353 */
3354DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3355{
3356 NOREF(pMixedCtx);
3357
3358 int rc = VINF_SUCCESS;
3359 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3360 {
3361 PVM pVM = pVCpu->CTX_SUFF(pVM);
3362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3364
3365 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3366 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3367
3368 /*
3369 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3370 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3371 */
3372#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3373 if (HMVMX_IS_64BIT_HOST_MODE())
3374 {
3375 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3376 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3377 }
3378 else
3379 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3380#else
3381 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3382 {
3383 /* The switcher returns to long mode, EFER is managed by the switcher. */
3384 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3385 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3386 }
3387 else
3388 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3389#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3390
3391 /* If the newer VMCS fields for managing EFER exists, use it. */
3392 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3393 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3394 {
3395 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3396 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3397 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3398 }
3399
3400 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3401 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3402
3403 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3404 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3405 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3406
3407 if ( pVM->hm.s.vmx.fUsePreemptTimer
3408 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3409 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3410
3411 if ((val & zap) != val)
3412 {
3413 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3414 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3415 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3416 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3417 }
3418
3419 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3420 AssertRCReturn(rc, rc);
3421
3422 pVCpu->hm.s.vmx.u32ExitCtls = val;
3423 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3424 }
3425 return rc;
3426}
3427
3428
3429/**
3430 * Loads the guest APIC and related state.
3431 *
3432 * @returns VBox status code.
3433 * @param pVM Pointer to the VM.
3434 * @param pVCpu Pointer to the VMCPU.
3435 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3436 * out-of-sync. Make sure to update the required fields
3437 * before using them.
3438 */
3439DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3440{
3441 NOREF(pMixedCtx);
3442
3443 int rc = VINF_SUCCESS;
3444 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3445 {
3446 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3447 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3448 {
3449 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3450
3451 bool fPendingIntr = false;
3452 uint8_t u8Tpr = 0;
3453 uint8_t u8PendingIntr = 0;
3454 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3455 AssertRCReturn(rc, rc);
3456
3457 /*
3458 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3459 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3460 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3461 * the interrupt when we VM-exit for other reasons.
3462 */
3463 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3464 uint32_t u32TprThreshold = 0;
3465 if (fPendingIntr)
3466 {
3467 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3468 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3469 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3470 if (u8PendingPriority <= u8TprPriority)
3471 u32TprThreshold = u8PendingPriority;
3472 else
3473 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3474 }
3475 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3476
3477 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3478 AssertRCReturn(rc, rc);
3479 }
3480
3481 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3482 }
3483 return rc;
3484}
3485
3486
3487/**
3488 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3489 *
3490 * @returns Guest's interruptibility-state.
3491 * @param pVCpu Pointer to the VMCPU.
3492 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3493 * out-of-sync. Make sure to update the required fields
3494 * before using them.
3495 *
3496 * @remarks No-long-jump zone!!!
3497 */
3498DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3499{
3500 /*
3501 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3502 */
3503 uint32_t uIntrState = 0;
3504 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3505 {
3506 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3507 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3508 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3509 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3510 {
3511 if (pMixedCtx->eflags.Bits.u1IF)
3512 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3513 else
3514 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3515 }
3516 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3517 }
3518
3519 /*
3520 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3521 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3522 * setting this would block host-NMIs and IRET will not clear the blocking.
3523 *
3524 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3525 */
3526 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3527 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3528 {
3529 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3530 }
3531
3532 return uIntrState;
3533}
3534
3535
3536/**
3537 * Loads the guest's interruptibility-state into the guest-state area in the
3538 * VMCS.
3539 *
3540 * @returns VBox status code.
3541 * @param pVCpu Pointer to the VMCPU.
3542 * @param uIntrState The interruptibility-state to set.
3543 */
3544static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3545{
3546 NOREF(pVCpu);
3547 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3548 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3549 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3550 AssertRCReturn(rc, rc);
3551 return rc;
3552}
3553
3554
3555/**
3556 * Loads the exception intercepts required for guest execution in the VMCS.
3557 *
3558 * @returns VBox status code.
3559 * @param pVCpu Pointer to the VMCPU.
3560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3561 * out-of-sync. Make sure to update the required fields
3562 * before using them.
3563 */
3564static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3565{
3566 NOREF(pMixedCtx);
3567 int rc = VINF_SUCCESS;
3568 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3569 {
3570 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3571 if (pVCpu->hm.s.fGIMTrapXcptUD)
3572 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3573 else
3574 {
3575#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3576 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3577#endif
3578 }
3579
3580 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3581 AssertRCReturn(rc, rc);
3582
3583 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3584 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3585 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3586 }
3587 return rc;
3588}
3589
3590
3591/**
3592 * Loads the guest's RIP into the guest-state area in the VMCS.
3593 *
3594 * @returns VBox status code.
3595 * @param pVCpu Pointer to the VMCPU.
3596 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3597 * out-of-sync. Make sure to update the required fields
3598 * before using them.
3599 *
3600 * @remarks No-long-jump zone!!!
3601 */
3602static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3603{
3604 int rc = VINF_SUCCESS;
3605 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3606 {
3607 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3608 AssertRCReturn(rc, rc);
3609
3610 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3611 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3612 HMCPU_CF_VALUE(pVCpu)));
3613 }
3614 return rc;
3615}
3616
3617
3618/**
3619 * Loads the guest's RSP into the guest-state area in the VMCS.
3620 *
3621 * @returns VBox status code.
3622 * @param pVCpu Pointer to the VMCPU.
3623 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3624 * out-of-sync. Make sure to update the required fields
3625 * before using them.
3626 *
3627 * @remarks No-long-jump zone!!!
3628 */
3629static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3630{
3631 int rc = VINF_SUCCESS;
3632 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3633 {
3634 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3635 AssertRCReturn(rc, rc);
3636
3637 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3638 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3639 }
3640 return rc;
3641}
3642
3643
3644/**
3645 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3646 *
3647 * @returns VBox status code.
3648 * @param pVCpu Pointer to the VMCPU.
3649 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3650 * out-of-sync. Make sure to update the required fields
3651 * before using them.
3652 *
3653 * @remarks No-long-jump zone!!!
3654 */
3655static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3656{
3657 int rc = VINF_SUCCESS;
3658 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3659 {
3660 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3661 Let us assert it as such and use 32-bit VMWRITE. */
3662 Assert(!(pMixedCtx->rflags.u64 >> 32));
3663 X86EFLAGS Eflags = pMixedCtx->eflags;
3664 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3665 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3666 * These will never be cleared/set, unless some other part of the VMM
3667 * code is buggy - in which case we're better of finding and fixing
3668 * those bugs than hiding them. */
3669 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3670 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3671 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3672 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3673
3674 /*
3675 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3676 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3677 */
3678 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3679 {
3680 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3681 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3682 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3683 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3684 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3685 }
3686
3687 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3688 AssertRCReturn(rc, rc);
3689
3690 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3691 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3692 }
3693 return rc;
3694}
3695
3696
3697/**
3698 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3699 *
3700 * @returns VBox status code.
3701 * @param pVCpu Pointer to the VMCPU.
3702 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3703 * out-of-sync. Make sure to update the required fields
3704 * before using them.
3705 *
3706 * @remarks No-long-jump zone!!!
3707 */
3708DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3709{
3710 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3711 AssertRCReturn(rc, rc);
3712 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3713 AssertRCReturn(rc, rc);
3714 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3715 AssertRCReturn(rc, rc);
3716 return rc;
3717}
3718
3719
3720/**
3721 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3722 * CR0 is partially shared with the host and we have to consider the FPU bits.
3723 *
3724 * @returns VBox status code.
3725 * @param pVM Pointer to the VM.
3726 * @param pVCpu Pointer to the VMCPU.
3727 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3728 * out-of-sync. Make sure to update the required fields
3729 * before using them.
3730 *
3731 * @remarks No-long-jump zone!!!
3732 */
3733static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3734{
3735 /*
3736 * Guest CR0.
3737 * Guest FPU.
3738 */
3739 int rc = VINF_SUCCESS;
3740 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3741 {
3742 Assert(!(pMixedCtx->cr0 >> 32));
3743 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3744 PVM pVM = pVCpu->CTX_SUFF(pVM);
3745
3746 /* The guest's view (read access) of its CR0 is unblemished. */
3747 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3748 AssertRCReturn(rc, rc);
3749 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3750
3751 /* Setup VT-x's view of the guest CR0. */
3752 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3753 if (pVM->hm.s.fNestedPaging)
3754 {
3755 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3756 {
3757 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3758 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3759 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3760 }
3761 else
3762 {
3763 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3764 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3765 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3766 }
3767
3768 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3769 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3770 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3771
3772 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3773 AssertRCReturn(rc, rc);
3774 }
3775 else
3776 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3777
3778 /*
3779 * Guest FPU bits.
3780 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3781 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3782 */
3783 u32GuestCR0 |= X86_CR0_NE;
3784 bool fInterceptNM = false;
3785 if (CPUMIsGuestFPUStateActive(pVCpu))
3786 {
3787 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3788 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3789 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3790 }
3791 else
3792 {
3793 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3794 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3795 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3796 }
3797
3798 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3799 bool fInterceptMF = false;
3800 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3801 fInterceptMF = true;
3802
3803 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3804 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3805 {
3806 Assert(PDMVmmDevHeapIsEnabled(pVM));
3807 Assert(pVM->hm.s.vmx.pRealModeTSS);
3808 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3809 fInterceptNM = true;
3810 fInterceptMF = true;
3811 }
3812 else
3813 {
3814 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626} comment #11. */
3815 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3816 }
3817 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3818
3819 if (fInterceptNM)
3820 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3821 else
3822 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3823
3824 if (fInterceptMF)
3825 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3826 else
3827 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3828
3829 /* Additional intercepts for debugging, define these yourself explicitly. */
3830#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3831 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3832 | RT_BIT(X86_XCPT_BP)
3833 | RT_BIT(X86_XCPT_DB)
3834 | RT_BIT(X86_XCPT_DE)
3835 | RT_BIT(X86_XCPT_NM)
3836 | RT_BIT(X86_XCPT_TS)
3837 | RT_BIT(X86_XCPT_UD)
3838 | RT_BIT(X86_XCPT_NP)
3839 | RT_BIT(X86_XCPT_SS)
3840 | RT_BIT(X86_XCPT_GP)
3841 | RT_BIT(X86_XCPT_PF)
3842 | RT_BIT(X86_XCPT_MF)
3843 ;
3844#elif defined(HMVMX_ALWAYS_TRAP_PF)
3845 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3846#endif
3847
3848 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3849
3850 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3851 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3852 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3853 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3854 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3855 else
3856 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3857
3858 u32GuestCR0 |= uSetCR0;
3859 u32GuestCR0 &= uZapCR0;
3860 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3861
3862 /* Write VT-x's view of the guest CR0 into the VMCS. */
3863 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3864 AssertRCReturn(rc, rc);
3865 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3866 uZapCR0));
3867
3868 /*
3869 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3870 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3871 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3872 */
3873 uint32_t u32CR0Mask = 0;
3874 u32CR0Mask = X86_CR0_PE
3875 | X86_CR0_NE
3876 | X86_CR0_WP
3877 | X86_CR0_PG
3878 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3879 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3880 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3881
3882 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3883 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3884 * and @bugref{6944}. */
3885#if 0
3886 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3887 u32CR0Mask &= ~X86_CR0_PE;
3888#endif
3889 if (pVM->hm.s.fNestedPaging)
3890 u32CR0Mask &= ~X86_CR0_WP;
3891
3892 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3893 if (fInterceptNM)
3894 {
3895 u32CR0Mask |= X86_CR0_TS
3896 | X86_CR0_MP;
3897 }
3898
3899 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3900 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3901 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3902 AssertRCReturn(rc, rc);
3903 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3904
3905 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3906 }
3907 return rc;
3908}
3909
3910
3911/**
3912 * Loads the guest control registers (CR3, CR4) into the guest-state area
3913 * in the VMCS.
3914 *
3915 * @returns VBox status code.
3916 * @param pVM Pointer to the VM.
3917 * @param pVCpu Pointer to the VMCPU.
3918 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3919 * out-of-sync. Make sure to update the required fields
3920 * before using them.
3921 *
3922 * @remarks No-long-jump zone!!!
3923 */
3924static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3925{
3926 int rc = VINF_SUCCESS;
3927 PVM pVM = pVCpu->CTX_SUFF(pVM);
3928
3929 /*
3930 * Guest CR2.
3931 * It's always loaded in the assembler code. Nothing to do here.
3932 */
3933
3934 /*
3935 * Guest CR3.
3936 */
3937 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3938 {
3939 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3940 if (pVM->hm.s.fNestedPaging)
3941 {
3942 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3943
3944 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3945 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3946 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3947 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3948
3949 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3950 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3951 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3952
3953 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3954 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3955 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3956 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3957
3958 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3959 AssertRCReturn(rc, rc);
3960 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3961
3962 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3963 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3964 {
3965 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3966 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3967 {
3968 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3969 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3970 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3971 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3972 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3973 }
3974
3975 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3976 have Unrestricted Execution to handle the guest when it's not using paging. */
3977 GCPhysGuestCR3 = pMixedCtx->cr3;
3978 }
3979 else
3980 {
3981 /*
3982 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3983 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3984 * EPT takes care of translating it to host-physical addresses.
3985 */
3986 RTGCPHYS GCPhys;
3987 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3988 Assert(PDMVmmDevHeapIsEnabled(pVM));
3989
3990 /* We obtain it here every time as the guest could have relocated this PCI region. */
3991 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3992 AssertRCReturn(rc, rc);
3993
3994 GCPhysGuestCR3 = GCPhys;
3995 }
3996
3997 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3998 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3999 }
4000 else
4001 {
4002 /* Non-nested paging case, just use the hypervisor's CR3. */
4003 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4004
4005 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4006 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4007 }
4008 AssertRCReturn(rc, rc);
4009
4010 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4011 }
4012
4013 /*
4014 * Guest CR4.
4015 */
4016 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4017 {
4018 Assert(!(pMixedCtx->cr4 >> 32));
4019 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4020
4021 /* The guest's view of its CR4 is unblemished. */
4022 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4023 AssertRCReturn(rc, rc);
4024 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4025
4026 /* Setup VT-x's view of the guest CR4. */
4027 /*
4028 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4029 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4030 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4031 */
4032 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4033 {
4034 Assert(pVM->hm.s.vmx.pRealModeTSS);
4035 Assert(PDMVmmDevHeapIsEnabled(pVM));
4036 u32GuestCR4 &= ~X86_CR4_VME;
4037 }
4038
4039 if (pVM->hm.s.fNestedPaging)
4040 {
4041 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4042 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4043 {
4044 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4045 u32GuestCR4 |= X86_CR4_PSE;
4046 /* Our identity mapping is a 32-bit page directory. */
4047 u32GuestCR4 &= ~X86_CR4_PAE;
4048 }
4049 /* else use guest CR4.*/
4050 }
4051 else
4052 {
4053 /*
4054 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4055 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4056 */
4057 switch (pVCpu->hm.s.enmShadowMode)
4058 {
4059 case PGMMODE_REAL: /* Real-mode. */
4060 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4061 case PGMMODE_32_BIT: /* 32-bit paging. */
4062 {
4063 u32GuestCR4 &= ~X86_CR4_PAE;
4064 break;
4065 }
4066
4067 case PGMMODE_PAE: /* PAE paging. */
4068 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4069 {
4070 u32GuestCR4 |= X86_CR4_PAE;
4071 break;
4072 }
4073
4074 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4075 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4076#ifdef VBOX_ENABLE_64_BITS_GUESTS
4077 break;
4078#endif
4079 default:
4080 AssertFailed();
4081 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4082 }
4083 }
4084
4085 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4086 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4087 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4088 u32GuestCR4 |= uSetCR4;
4089 u32GuestCR4 &= uZapCR4;
4090
4091 /* Write VT-x's view of the guest CR4 into the VMCS. */
4092 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4093 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4094 AssertRCReturn(rc, rc);
4095
4096 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4097 uint32_t u32CR4Mask = 0;
4098 u32CR4Mask = X86_CR4_VME
4099 | X86_CR4_PAE
4100 | X86_CR4_PGE
4101 | X86_CR4_PSE
4102 | X86_CR4_VMXE;
4103 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4104 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4105 AssertRCReturn(rc, rc);
4106
4107 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4108 }
4109 return rc;
4110}
4111
4112
4113/**
4114 * Loads the guest debug registers into the guest-state area in the VMCS.
4115 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4116 *
4117 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4118 *
4119 * @returns VBox status code.
4120 * @param pVCpu Pointer to the VMCPU.
4121 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4122 * out-of-sync. Make sure to update the required fields
4123 * before using them.
4124 *
4125 * @remarks No-long-jump zone!!!
4126 */
4127static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4128{
4129 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4130 return VINF_SUCCESS;
4131
4132#ifdef VBOX_STRICT
4133 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4134 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4135 {
4136 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4137 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4138 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4139 }
4140#endif
4141
4142 int rc;
4143 PVM pVM = pVCpu->CTX_SUFF(pVM);
4144 bool fInterceptDB = false;
4145 bool fInterceptMovDRx = false;
4146 if ( pVCpu->hm.s.fSingleInstruction
4147 || DBGFIsStepping(pVCpu))
4148 {
4149 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4150 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4151 {
4152 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4153 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4154 AssertRCReturn(rc, rc);
4155 Assert(fInterceptDB == false);
4156 }
4157 else
4158 {
4159 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4160 pVCpu->hm.s.fClearTrapFlag = true;
4161 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4162 fInterceptDB = true;
4163 }
4164 }
4165
4166 if ( fInterceptDB
4167 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4168 {
4169 /*
4170 * Use the combined guest and host DRx values found in the hypervisor
4171 * register set because the debugger has breakpoints active or someone
4172 * is single stepping on the host side without a monitor trap flag.
4173 *
4174 * Note! DBGF expects a clean DR6 state before executing guest code.
4175 */
4176#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4177 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4178 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4179 {
4180 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4181 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4182 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4183 }
4184 else
4185#endif
4186 if (!CPUMIsHyperDebugStateActive(pVCpu))
4187 {
4188 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4189 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4190 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4191 }
4192
4193 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4194 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4195 AssertRCReturn(rc, rc);
4196
4197 pVCpu->hm.s.fUsingHyperDR7 = true;
4198 fInterceptDB = true;
4199 fInterceptMovDRx = true;
4200 }
4201 else
4202 {
4203 /*
4204 * If the guest has enabled debug registers, we need to load them prior to
4205 * executing guest code so they'll trigger at the right time.
4206 */
4207 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4208 {
4209#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4210 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4211 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4212 {
4213 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4214 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4215 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4216 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4217 }
4218 else
4219#endif
4220 if (!CPUMIsGuestDebugStateActive(pVCpu))
4221 {
4222 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4223 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4224 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4225 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4226 }
4227 Assert(!fInterceptDB);
4228 Assert(!fInterceptMovDRx);
4229 }
4230 /*
4231 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4232 * must intercept #DB in order to maintain a correct DR6 guest value.
4233 */
4234#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4235 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4236 && !CPUMIsGuestDebugStateActive(pVCpu))
4237#else
4238 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4239#endif
4240 {
4241 fInterceptMovDRx = true;
4242 fInterceptDB = true;
4243 }
4244
4245 /* Update guest DR7. */
4246 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4247 AssertRCReturn(rc, rc);
4248
4249 pVCpu->hm.s.fUsingHyperDR7 = false;
4250 }
4251
4252 /*
4253 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4254 */
4255 if ( fInterceptDB
4256 || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4257 {
4258 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4259 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
4260 }
4261 else
4262 {
4263#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4264 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4265 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
4266#endif
4267 }
4268
4269 /*
4270 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4271 */
4272 if (fInterceptMovDRx)
4273 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4274 else
4275 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4276 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4277 AssertRCReturn(rc, rc);
4278
4279 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4280 return VINF_SUCCESS;
4281}
4282
4283
4284#ifdef VBOX_STRICT
4285/**
4286 * Strict function to validate segment registers.
4287 *
4288 * @remarks ASSUMES CR0 is up to date.
4289 */
4290static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4291{
4292 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4293 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4294 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4295 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4296 && ( !CPUMIsGuestInRealModeEx(pCtx)
4297 && !CPUMIsGuestInV86ModeEx(pCtx)))
4298 {
4299 /* Protected mode checks */
4300 /* CS */
4301 Assert(pCtx->cs.Attr.n.u1Present);
4302 Assert(!(pCtx->cs.Attr.u & 0xf00));
4303 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4304 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4305 || !(pCtx->cs.Attr.n.u1Granularity));
4306 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4307 || (pCtx->cs.Attr.n.u1Granularity));
4308 /* CS cannot be loaded with NULL in protected mode. */
4309 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4310 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4311 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4312 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4313 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4314 else
4315 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4316 /* SS */
4317 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4318 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4319 if ( !(pCtx->cr0 & X86_CR0_PE)
4320 || pCtx->cs.Attr.n.u4Type == 3)
4321 {
4322 Assert(!pCtx->ss.Attr.n.u2Dpl);
4323 }
4324 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4325 {
4326 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4327 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4328 Assert(pCtx->ss.Attr.n.u1Present);
4329 Assert(!(pCtx->ss.Attr.u & 0xf00));
4330 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4331 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4332 || !(pCtx->ss.Attr.n.u1Granularity));
4333 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4334 || (pCtx->ss.Attr.n.u1Granularity));
4335 }
4336 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4337 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4338 {
4339 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4340 Assert(pCtx->ds.Attr.n.u1Present);
4341 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4342 Assert(!(pCtx->ds.Attr.u & 0xf00));
4343 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4344 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4345 || !(pCtx->ds.Attr.n.u1Granularity));
4346 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4347 || (pCtx->ds.Attr.n.u1Granularity));
4348 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4349 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4350 }
4351 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4352 {
4353 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4354 Assert(pCtx->es.Attr.n.u1Present);
4355 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4356 Assert(!(pCtx->es.Attr.u & 0xf00));
4357 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4358 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4359 || !(pCtx->es.Attr.n.u1Granularity));
4360 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4361 || (pCtx->es.Attr.n.u1Granularity));
4362 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4363 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4364 }
4365 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4366 {
4367 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4368 Assert(pCtx->fs.Attr.n.u1Present);
4369 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4370 Assert(!(pCtx->fs.Attr.u & 0xf00));
4371 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4372 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4373 || !(pCtx->fs.Attr.n.u1Granularity));
4374 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4375 || (pCtx->fs.Attr.n.u1Granularity));
4376 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4377 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4378 }
4379 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4380 {
4381 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4382 Assert(pCtx->gs.Attr.n.u1Present);
4383 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4384 Assert(!(pCtx->gs.Attr.u & 0xf00));
4385 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4386 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4387 || !(pCtx->gs.Attr.n.u1Granularity));
4388 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4389 || (pCtx->gs.Attr.n.u1Granularity));
4390 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4391 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4392 }
4393 /* 64-bit capable CPUs. */
4394# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4395 if (HMVMX_IS_64BIT_HOST_MODE())
4396 {
4397 Assert(!(pCtx->cs.u64Base >> 32));
4398 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4399 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4400 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4401 }
4402# endif
4403 }
4404 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4405 || ( CPUMIsGuestInRealModeEx(pCtx)
4406 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4407 {
4408 /* Real and v86 mode checks. */
4409 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4410 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4411 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4412 {
4413 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4414 }
4415 else
4416 {
4417 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4418 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4419 }
4420
4421 /* CS */
4422 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4423 Assert(pCtx->cs.u32Limit == 0xffff);
4424 Assert(u32CSAttr == 0xf3);
4425 /* SS */
4426 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4427 Assert(pCtx->ss.u32Limit == 0xffff);
4428 Assert(u32SSAttr == 0xf3);
4429 /* DS */
4430 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4431 Assert(pCtx->ds.u32Limit == 0xffff);
4432 Assert(u32DSAttr == 0xf3);
4433 /* ES */
4434 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4435 Assert(pCtx->es.u32Limit == 0xffff);
4436 Assert(u32ESAttr == 0xf3);
4437 /* FS */
4438 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4439 Assert(pCtx->fs.u32Limit == 0xffff);
4440 Assert(u32FSAttr == 0xf3);
4441 /* GS */
4442 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4443 Assert(pCtx->gs.u32Limit == 0xffff);
4444 Assert(u32GSAttr == 0xf3);
4445 /* 64-bit capable CPUs. */
4446# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4447 if (HMVMX_IS_64BIT_HOST_MODE())
4448 {
4449 Assert(!(pCtx->cs.u64Base >> 32));
4450 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4451 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4452 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4453 }
4454# endif
4455 }
4456}
4457#endif /* VBOX_STRICT */
4458
4459
4460/**
4461 * Writes a guest segment register into the guest-state area in the VMCS.
4462 *
4463 * @returns VBox status code.
4464 * @param pVCpu Pointer to the VMCPU.
4465 * @param idxSel Index of the selector in the VMCS.
4466 * @param idxLimit Index of the segment limit in the VMCS.
4467 * @param idxBase Index of the segment base in the VMCS.
4468 * @param idxAccess Index of the access rights of the segment in the VMCS.
4469 * @param pSelReg Pointer to the segment selector.
4470 *
4471 * @remarks No-long-jump zone!!!
4472 */
4473static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4474 uint32_t idxAccess, PCPUMSELREG pSelReg)
4475{
4476 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4477 AssertRCReturn(rc, rc);
4478 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4479 AssertRCReturn(rc, rc);
4480 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4481 AssertRCReturn(rc, rc);
4482
4483 uint32_t u32Access = pSelReg->Attr.u;
4484 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4485 {
4486 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4487 u32Access = 0xf3;
4488 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4489 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4490 }
4491 else
4492 {
4493 /*
4494 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4495 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4496 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4497 * loaded in protected-mode have their attribute as 0.
4498 */
4499 if (!u32Access)
4500 u32Access = X86DESCATTR_UNUSABLE;
4501 }
4502
4503 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4504 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4505 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4506
4507 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4508 AssertRCReturn(rc, rc);
4509 return rc;
4510}
4511
4512
4513/**
4514 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4515 * into the guest-state area in the VMCS.
4516 *
4517 * @returns VBox status code.
4518 * @param pVM Pointer to the VM.
4519 * @param pVCPU Pointer to the VMCPU.
4520 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4521 * out-of-sync. Make sure to update the required fields
4522 * before using them.
4523 *
4524 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4525 * @remarks No-long-jump zone!!!
4526 */
4527static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4528{
4529 int rc = VERR_INTERNAL_ERROR_5;
4530 PVM pVM = pVCpu->CTX_SUFF(pVM);
4531
4532 /*
4533 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4534 */
4535 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4536 {
4537 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4538 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4539 {
4540 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4541 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4542 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4543 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4544 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4545 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4546 }
4547
4548#ifdef VBOX_WITH_REM
4549 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4550 {
4551 Assert(pVM->hm.s.vmx.pRealModeTSS);
4552 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4553 if ( pVCpu->hm.s.vmx.fWasInRealMode
4554 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4555 {
4556 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4557 in real-mode (e.g. OpenBSD 4.0) */
4558 REMFlushTBs(pVM);
4559 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4560 pVCpu->hm.s.vmx.fWasInRealMode = false;
4561 }
4562 }
4563#endif
4564 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4565 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4566 AssertRCReturn(rc, rc);
4567 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4568 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4569 AssertRCReturn(rc, rc);
4570 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4571 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4572 AssertRCReturn(rc, rc);
4573 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4574 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4575 AssertRCReturn(rc, rc);
4576 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4577 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4578 AssertRCReturn(rc, rc);
4579 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4580 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4581 AssertRCReturn(rc, rc);
4582
4583#ifdef VBOX_STRICT
4584 /* Validate. */
4585 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4586#endif
4587
4588 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4589 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4590 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4591 }
4592
4593 /*
4594 * Guest TR.
4595 */
4596 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4597 {
4598 /*
4599 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4600 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4601 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4602 */
4603 uint16_t u16Sel = 0;
4604 uint32_t u32Limit = 0;
4605 uint64_t u64Base = 0;
4606 uint32_t u32AccessRights = 0;
4607
4608 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4609 {
4610 u16Sel = pMixedCtx->tr.Sel;
4611 u32Limit = pMixedCtx->tr.u32Limit;
4612 u64Base = pMixedCtx->tr.u64Base;
4613 u32AccessRights = pMixedCtx->tr.Attr.u;
4614 }
4615 else
4616 {
4617 Assert(pVM->hm.s.vmx.pRealModeTSS);
4618 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4619
4620 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4621 RTGCPHYS GCPhys;
4622 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4623 AssertRCReturn(rc, rc);
4624
4625 X86DESCATTR DescAttr;
4626 DescAttr.u = 0;
4627 DescAttr.n.u1Present = 1;
4628 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4629
4630 u16Sel = 0;
4631 u32Limit = HM_VTX_TSS_SIZE;
4632 u64Base = GCPhys; /* in real-mode phys = virt. */
4633 u32AccessRights = DescAttr.u;
4634 }
4635
4636 /* Validate. */
4637 Assert(!(u16Sel & RT_BIT(2)));
4638 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4639 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4640 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4641 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4642 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4643 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4644 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4645 Assert( (u32Limit & 0xfff) == 0xfff
4646 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4647 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4648 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4649
4650 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4651 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4652 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4653 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4654
4655 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4656 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4657 }
4658
4659 /*
4660 * Guest GDTR.
4661 */
4662 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4663 {
4664 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4665 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4666
4667 /* Validate. */
4668 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4669
4670 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4671 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4672 }
4673
4674 /*
4675 * Guest LDTR.
4676 */
4677 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4678 {
4679 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4680 uint32_t u32Access = 0;
4681 if (!pMixedCtx->ldtr.Attr.u)
4682 u32Access = X86DESCATTR_UNUSABLE;
4683 else
4684 u32Access = pMixedCtx->ldtr.Attr.u;
4685
4686 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4687 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4688 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4689 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4690
4691 /* Validate. */
4692 if (!(u32Access & X86DESCATTR_UNUSABLE))
4693 {
4694 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4695 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4696 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4697 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4698 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4699 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4700 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4701 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4702 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4703 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4704 }
4705
4706 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4707 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4708 }
4709
4710 /*
4711 * Guest IDTR.
4712 */
4713 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4714 {
4715 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4716 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4717
4718 /* Validate. */
4719 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4720
4721 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4722 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4723 }
4724
4725 return VINF_SUCCESS;
4726}
4727
4728
4729/**
4730 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4731 * areas.
4732 *
4733 * These MSRs will automatically be loaded to the host CPU on every successful
4734 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4735 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4736 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4737 *
4738 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4739 *
4740 * @returns VBox status code.
4741 * @param pVCpu Pointer to the VMCPU.
4742 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4743 * out-of-sync. Make sure to update the required fields
4744 * before using them.
4745 *
4746 * @remarks No-long-jump zone!!!
4747 */
4748static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4749{
4750 AssertPtr(pVCpu);
4751 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4752
4753 /*
4754 * MSRs that we use the auto-load/store MSR area in the VMCS.
4755 */
4756 PVM pVM = pVCpu->CTX_SUFF(pVM);
4757 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4758 {
4759 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4760#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4761 if (pVM->hm.s.fAllow64BitGuests)
4762 {
4763 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4764 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4765 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4766 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4767# ifdef DEBUG
4768 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4769 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4770 {
4771 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4772 pMsr->u64Value));
4773 }
4774# endif
4775 }
4776#endif
4777 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4778 }
4779
4780 /*
4781 * Guest Sysenter MSRs.
4782 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4783 * VM-exits on WRMSRs for these MSRs.
4784 */
4785 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4786 {
4787 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4788 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4789 }
4790
4791 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4792 {
4793 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4794 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4795 }
4796
4797 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4798 {
4799 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4800 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4801 }
4802
4803 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4804 {
4805 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4806 {
4807 /*
4808 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4809 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4810 */
4811 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4812 {
4813 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4814 AssertRCReturn(rc,rc);
4815 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4816 }
4817 else
4818 {
4819 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4820 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4821 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4822 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4823 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4824 }
4825 }
4826 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4827 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4828 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4829 }
4830
4831 return VINF_SUCCESS;
4832}
4833
4834
4835/**
4836 * Loads the guest activity state into the guest-state area in the VMCS.
4837 *
4838 * @returns VBox status code.
4839 * @param pVCpu Pointer to the VMCPU.
4840 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4841 * out-of-sync. Make sure to update the required fields
4842 * before using them.
4843 *
4844 * @remarks No-long-jump zone!!!
4845 */
4846static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4847{
4848 NOREF(pCtx);
4849 /** @todo See if we can make use of other states, e.g.
4850 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4851 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4852 {
4853 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4854 AssertRCReturn(rc, rc);
4855
4856 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4857 }
4858 return VINF_SUCCESS;
4859}
4860
4861
4862/**
4863 * Sets up the appropriate function to run guest code.
4864 *
4865 * @returns VBox status code.
4866 * @param pVCpu Pointer to the VMCPU.
4867 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4868 * out-of-sync. Make sure to update the required fields
4869 * before using them.
4870 *
4871 * @remarks No-long-jump zone!!!
4872 */
4873static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4874{
4875 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4876 {
4877#ifndef VBOX_ENABLE_64_BITS_GUESTS
4878 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4879#endif
4880 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4881#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4882 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4883 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4884 {
4885 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4886 {
4887 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4888 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4889 | HM_CHANGED_VMX_ENTRY_CTLS
4890 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4891 }
4892 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4893 }
4894#else
4895 /* 64-bit host or hybrid host. */
4896 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4897#endif
4898 }
4899 else
4900 {
4901 /* Guest is not in long mode, use the 32-bit handler. */
4902#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4903 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4904 {
4905 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4906 {
4907 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4908 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4909 | HM_CHANGED_VMX_ENTRY_CTLS
4910 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4911 }
4912 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4913 }
4914#else
4915 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4916#endif
4917 }
4918 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4919 return VINF_SUCCESS;
4920}
4921
4922
4923/**
4924 * Wrapper for running the guest code in VT-x.
4925 *
4926 * @returns VBox strict status code.
4927 * @param pVM Pointer to the VM.
4928 * @param pVCpu Pointer to the VMCPU.
4929 * @param pCtx Pointer to the guest-CPU context.
4930 *
4931 * @remarks No-long-jump zone!!!
4932 */
4933DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4934{
4935 /*
4936 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4937 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4938 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4939 */
4940 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4941 /** @todo Add stats for resume vs launch. */
4942#ifdef VBOX_WITH_KERNEL_USING_XMM
4943 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4944#else
4945 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4946#endif
4947}
4948
4949
4950/**
4951 * Reports world-switch error and dumps some useful debug info.
4952 *
4953 * @param pVM Pointer to the VM.
4954 * @param pVCpu Pointer to the VMCPU.
4955 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4956 * @param pCtx Pointer to the guest-CPU context.
4957 * @param pVmxTransient Pointer to the VMX transient structure (only
4958 * exitReason updated).
4959 */
4960static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4961{
4962 Assert(pVM);
4963 Assert(pVCpu);
4964 Assert(pCtx);
4965 Assert(pVmxTransient);
4966 HMVMX_ASSERT_PREEMPT_SAFE();
4967
4968 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4969 switch (rcVMRun)
4970 {
4971 case VERR_VMX_INVALID_VMXON_PTR:
4972 AssertFailed();
4973 break;
4974 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4975 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4976 {
4977 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4978 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4979 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4980 AssertRC(rc);
4981
4982 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4983 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4984 Cannot do it here as we may have been long preempted. */
4985
4986#ifdef VBOX_STRICT
4987 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4988 pVmxTransient->uExitReason));
4989 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4990 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4991 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4992 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4993 else
4994 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4995 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4996 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4997
4998 /* VMX control bits. */
4999 uint32_t u32Val;
5000 uint64_t u64Val;
5001 HMVMXHCUINTREG uHCReg;
5002 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5003 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5004 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5005 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5006 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5007 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5008 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5009 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5010 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5011 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5012 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5013 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5014 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5015 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5016 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5017 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5018 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5019 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5020 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5021 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5022 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5023 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5024 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5025 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5026 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5027 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5028 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5029 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5030 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5031 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5032 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5033 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5034 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5035 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5036 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5037 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5038 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5039 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5040 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5041 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5042 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5043 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5044
5045 /* Guest bits. */
5046 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5047 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5048 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5049 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5050 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5051 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5052 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5053 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5054
5055 /* Host bits. */
5056 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5057 Log4(("Host CR0 %#RHr\n", uHCReg));
5058 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5059 Log4(("Host CR3 %#RHr\n", uHCReg));
5060 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5061 Log4(("Host CR4 %#RHr\n", uHCReg));
5062
5063 RTGDTR HostGdtr;
5064 PCX86DESCHC pDesc;
5065 ASMGetGDTR(&HostGdtr);
5066 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5067 Log4(("Host CS %#08x\n", u32Val));
5068 if (u32Val < HostGdtr.cbGdt)
5069 {
5070 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5071 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5072 }
5073
5074 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5075 Log4(("Host DS %#08x\n", u32Val));
5076 if (u32Val < HostGdtr.cbGdt)
5077 {
5078 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5079 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5080 }
5081
5082 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5083 Log4(("Host ES %#08x\n", u32Val));
5084 if (u32Val < HostGdtr.cbGdt)
5085 {
5086 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5087 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5088 }
5089
5090 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5091 Log4(("Host FS %#08x\n", u32Val));
5092 if (u32Val < HostGdtr.cbGdt)
5093 {
5094 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5095 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5096 }
5097
5098 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5099 Log4(("Host GS %#08x\n", u32Val));
5100 if (u32Val < HostGdtr.cbGdt)
5101 {
5102 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5103 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5104 }
5105
5106 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5107 Log4(("Host SS %#08x\n", u32Val));
5108 if (u32Val < HostGdtr.cbGdt)
5109 {
5110 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5111 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5112 }
5113
5114 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5115 Log4(("Host TR %#08x\n", u32Val));
5116 if (u32Val < HostGdtr.cbGdt)
5117 {
5118 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5119 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5120 }
5121
5122 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5123 Log4(("Host TR Base %#RHv\n", uHCReg));
5124 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5125 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5126 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5127 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5128 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5129 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5130 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5131 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5132 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5133 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5134 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5135 Log4(("Host RSP %#RHv\n", uHCReg));
5136 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5137 Log4(("Host RIP %#RHv\n", uHCReg));
5138# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5139 if (HMVMX_IS_64BIT_HOST_MODE())
5140 {
5141 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5142 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5143 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5144 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5145 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5146 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5147 }
5148# endif
5149#endif /* VBOX_STRICT */
5150 break;
5151 }
5152
5153 default:
5154 /* Impossible */
5155 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5156 break;
5157 }
5158 NOREF(pVM); NOREF(pCtx);
5159}
5160
5161
5162#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5163#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5164# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5165#endif
5166#ifdef VBOX_STRICT
5167static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5168{
5169 switch (idxField)
5170 {
5171 case VMX_VMCS_GUEST_RIP:
5172 case VMX_VMCS_GUEST_RSP:
5173 case VMX_VMCS_GUEST_SYSENTER_EIP:
5174 case VMX_VMCS_GUEST_SYSENTER_ESP:
5175 case VMX_VMCS_GUEST_GDTR_BASE:
5176 case VMX_VMCS_GUEST_IDTR_BASE:
5177 case VMX_VMCS_GUEST_CS_BASE:
5178 case VMX_VMCS_GUEST_DS_BASE:
5179 case VMX_VMCS_GUEST_ES_BASE:
5180 case VMX_VMCS_GUEST_FS_BASE:
5181 case VMX_VMCS_GUEST_GS_BASE:
5182 case VMX_VMCS_GUEST_SS_BASE:
5183 case VMX_VMCS_GUEST_LDTR_BASE:
5184 case VMX_VMCS_GUEST_TR_BASE:
5185 case VMX_VMCS_GUEST_CR3:
5186 return true;
5187 }
5188 return false;
5189}
5190
5191static bool hmR0VmxIsValidReadField(uint32_t idxField)
5192{
5193 switch (idxField)
5194 {
5195 /* Read-only fields. */
5196 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5197 return true;
5198 }
5199 /* Remaining readable fields should also be writable. */
5200 return hmR0VmxIsValidWriteField(idxField);
5201}
5202#endif /* VBOX_STRICT */
5203
5204
5205/**
5206 * Executes the specified handler in 64-bit mode.
5207 *
5208 * @returns VBox status code.
5209 * @param pVM Pointer to the VM.
5210 * @param pVCpu Pointer to the VMCPU.
5211 * @param pCtx Pointer to the guest CPU context.
5212 * @param enmOp The operation to perform.
5213 * @param cbParam Number of parameters.
5214 * @param paParam Array of 32-bit parameters.
5215 */
5216VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5217 uint32_t *paParam)
5218{
5219 int rc, rc2;
5220 PHMGLOBALCPUINFO pCpu;
5221 RTHCPHYS HCPhysCpuPage;
5222 RTCCUINTREG uOldEflags;
5223
5224 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5225 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5226 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5227 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5228
5229#ifdef VBOX_STRICT
5230 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5231 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5232
5233 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5234 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5235#endif
5236
5237 /* Disable interrupts. */
5238 uOldEflags = ASMIntDisableFlags();
5239
5240#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5241 RTCPUID idHostCpu = RTMpCpuId();
5242 CPUMR0SetLApic(pVCpu, idHostCpu);
5243#endif
5244
5245 pCpu = HMR0GetCurrentCpu();
5246 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5247
5248 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5249 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5250
5251 /* Leave VMX Root Mode. */
5252 VMXDisable();
5253
5254 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5255
5256 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5257 CPUMSetHyperEIP(pVCpu, enmOp);
5258 for (int i = (int)cbParam - 1; i >= 0; i--)
5259 CPUMPushHyper(pVCpu, paParam[i]);
5260
5261 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5262
5263 /* Call the switcher. */
5264 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5265 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5266
5267 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5268 /* Make sure the VMX instructions don't cause #UD faults. */
5269 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5270
5271 /* Re-enter VMX Root Mode */
5272 rc2 = VMXEnable(HCPhysCpuPage);
5273 if (RT_FAILURE(rc2))
5274 {
5275 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5276 ASMSetFlags(uOldEflags);
5277 return rc2;
5278 }
5279
5280 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5281 AssertRC(rc2);
5282 Assert(!(ASMGetFlags() & X86_EFL_IF));
5283 ASMSetFlags(uOldEflags);
5284 return rc;
5285}
5286
5287
5288/**
5289 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5290 * supporting 64-bit guests.
5291 *
5292 * @returns VBox status code.
5293 * @param fResume Whether to VMLAUNCH or VMRESUME.
5294 * @param pCtx Pointer to the guest-CPU context.
5295 * @param pCache Pointer to the VMCS cache.
5296 * @param pVM Pointer to the VM.
5297 * @param pVCpu Pointer to the VMCPU.
5298 */
5299DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5300{
5301 uint32_t aParam[6];
5302 PHMGLOBALCPUINFO pCpu = NULL;
5303 RTHCPHYS HCPhysCpuPage = 0;
5304 int rc = VERR_INTERNAL_ERROR_5;
5305
5306 pCpu = HMR0GetCurrentCpu();
5307 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5308
5309#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5310 pCache->uPos = 1;
5311 pCache->interPD = PGMGetInterPaeCR3(pVM);
5312 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5313#endif
5314
5315#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5316 pCache->TestIn.HCPhysCpuPage = 0;
5317 pCache->TestIn.HCPhysVmcs = 0;
5318 pCache->TestIn.pCache = 0;
5319 pCache->TestOut.HCPhysVmcs = 0;
5320 pCache->TestOut.pCache = 0;
5321 pCache->TestOut.pCtx = 0;
5322 pCache->TestOut.eflags = 0;
5323#endif
5324
5325 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5326 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5327 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5328 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5329 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5330 aParam[5] = 0;
5331
5332#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5333 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5334 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5335#endif
5336 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5337
5338#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5339 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5340 Assert(pCtx->dr[4] == 10);
5341 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5342#endif
5343
5344#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5345 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5346 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5347 pVCpu->hm.s.vmx.HCPhysVmcs));
5348 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5349 pCache->TestOut.HCPhysVmcs));
5350 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5351 pCache->TestOut.pCache));
5352 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5353 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5354 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5355 pCache->TestOut.pCtx));
5356 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5357#endif
5358 return rc;
5359}
5360
5361
5362/**
5363 * Initialize the VMCS-Read cache.
5364 *
5365 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5366 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5367 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5368 * (those that have a 32-bit FULL & HIGH part).
5369 *
5370 * @returns VBox status code.
5371 * @param pVM Pointer to the VM.
5372 * @param pVCpu Pointer to the VMCPU.
5373 */
5374static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5375{
5376#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5377{ \
5378 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5379 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5380 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5381 ++cReadFields; \
5382}
5383
5384 AssertPtr(pVM);
5385 AssertPtr(pVCpu);
5386 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5387 uint32_t cReadFields = 0;
5388
5389 /*
5390 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5391 * and serve to indicate exceptions to the rules.
5392 */
5393
5394 /* Guest-natural selector base fields. */
5395#if 0
5396 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5399#endif
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5401 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5402 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5403 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5404 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5405 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5406 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5407 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5408 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5409 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5410 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5411 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5412#if 0
5413 /* Unused natural width guest-state fields. */
5414 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5415 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5416#endif
5417 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5418 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5419
5420 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5421#if 0
5422 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5423 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5424 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5425 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5426 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5427 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5431#endif
5432
5433 /* Natural width guest-state fields. */
5434 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5435#if 0
5436 /* Currently unused field. */
5437 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5438#endif
5439
5440 if (pVM->hm.s.fNestedPaging)
5441 {
5442 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5443 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5444 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5445 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5446 }
5447 else
5448 {
5449 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5450 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5451 }
5452
5453#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5454 return VINF_SUCCESS;
5455}
5456
5457
5458/**
5459 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5460 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5461 * darwin, running 64-bit guests).
5462 *
5463 * @returns VBox status code.
5464 * @param pVCpu Pointer to the VMCPU.
5465 * @param idxField The VMCS field encoding.
5466 * @param u64Val 16, 32 or 64-bit value.
5467 */
5468VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5469{
5470 int rc;
5471 switch (idxField)
5472 {
5473 /*
5474 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5475 */
5476 /* 64-bit Control fields. */
5477 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5478 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5479 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5480 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5481 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5482 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5483 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5484 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5485 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5486 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5487 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5488 case VMX_VMCS64_CTRL_EPTP_FULL:
5489 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5490 /* 64-bit Guest-state fields. */
5491 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5492 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5493 case VMX_VMCS64_GUEST_PAT_FULL:
5494 case VMX_VMCS64_GUEST_EFER_FULL:
5495 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5496 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5497 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5498 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5499 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5500 /* 64-bit Host-state fields. */
5501 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5502 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5503 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5504 {
5505 rc = VMXWriteVmcs32(idxField, u64Val);
5506 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5507 break;
5508 }
5509
5510 /*
5511 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5512 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5513 */
5514 /* Natural-width Guest-state fields. */
5515 case VMX_VMCS_GUEST_CR3:
5516 case VMX_VMCS_GUEST_ES_BASE:
5517 case VMX_VMCS_GUEST_CS_BASE:
5518 case VMX_VMCS_GUEST_SS_BASE:
5519 case VMX_VMCS_GUEST_DS_BASE:
5520 case VMX_VMCS_GUEST_FS_BASE:
5521 case VMX_VMCS_GUEST_GS_BASE:
5522 case VMX_VMCS_GUEST_LDTR_BASE:
5523 case VMX_VMCS_GUEST_TR_BASE:
5524 case VMX_VMCS_GUEST_GDTR_BASE:
5525 case VMX_VMCS_GUEST_IDTR_BASE:
5526 case VMX_VMCS_GUEST_RSP:
5527 case VMX_VMCS_GUEST_RIP:
5528 case VMX_VMCS_GUEST_SYSENTER_ESP:
5529 case VMX_VMCS_GUEST_SYSENTER_EIP:
5530 {
5531 if (!(u64Val >> 32))
5532 {
5533 /* If this field is 64-bit, VT-x will zero out the top bits. */
5534 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5535 }
5536 else
5537 {
5538 /* Assert that only the 32->64 switcher case should ever come here. */
5539 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5540 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5541 }
5542 break;
5543 }
5544
5545 default:
5546 {
5547 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5548 rc = VERR_INVALID_PARAMETER;
5549 break;
5550 }
5551 }
5552 AssertRCReturn(rc, rc);
5553 return rc;
5554}
5555
5556
5557/**
5558 * Queue up a VMWRITE by using the VMCS write cache.
5559 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5560 *
5561 * @param pVCpu Pointer to the VMCPU.
5562 * @param idxField The VMCS field encoding.
5563 * @param u64Val 16, 32 or 64-bit value.
5564 */
5565VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5566{
5567 AssertPtr(pVCpu);
5568 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5569
5570 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5571 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5572
5573 /* Make sure there are no duplicates. */
5574 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5575 {
5576 if (pCache->Write.aField[i] == idxField)
5577 {
5578 pCache->Write.aFieldVal[i] = u64Val;
5579 return VINF_SUCCESS;
5580 }
5581 }
5582
5583 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5584 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5585 pCache->Write.cValidEntries++;
5586 return VINF_SUCCESS;
5587}
5588
5589/* Enable later when the assembly code uses these as callbacks. */
5590#if 0
5591/*
5592 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5593 *
5594 * @param pVCpu Pointer to the VMCPU.
5595 * @param pCache Pointer to the VMCS cache.
5596 *
5597 * @remarks No-long-jump zone!!!
5598 */
5599VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5600{
5601 AssertPtr(pCache);
5602 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5603 {
5604 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5605 AssertRC(rc);
5606 }
5607 pCache->Write.cValidEntries = 0;
5608}
5609
5610
5611/**
5612 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5613 *
5614 * @param pVCpu Pointer to the VMCPU.
5615 * @param pCache Pointer to the VMCS cache.
5616 *
5617 * @remarks No-long-jump zone!!!
5618 */
5619VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5620{
5621 AssertPtr(pCache);
5622 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5623 {
5624 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5625 AssertRC(rc);
5626 }
5627}
5628#endif
5629#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5630
5631
5632/**
5633 * Sets up the usage of TSC-offsetting and updates the VMCS.
5634 *
5635 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5636 * VMX preemption timer.
5637 *
5638 * @returns VBox status code.
5639 * @param pVM Pointer to the cross context VM structure.
5640 * @param pVCpu Pointer to the VMCPU.
5641 *
5642 * @remarks No-long-jump zone!!!
5643 */
5644static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5645{
5646 int rc;
5647 bool fOffsettedTsc;
5648 bool fParavirtTsc;
5649 if (pVM->hm.s.vmx.fUsePreemptTimer)
5650 {
5651 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5652 &fOffsettedTsc, &fParavirtTsc);
5653
5654 /* Make sure the returned values have sane upper and lower boundaries. */
5655 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5656 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5657 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5658 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5659
5660 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5661 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5662 }
5663 else
5664 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5665
5666 /** @todo later optimize this to be done elsewhere and not before every
5667 * VM-entry. */
5668 if (fParavirtTsc)
5669 {
5670 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5671 AssertRC(rc);
5672 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5673 }
5674
5675 if (fOffsettedTsc)
5676 {
5677 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5678 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5679
5680 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5681 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5682 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5683 }
5684 else
5685 {
5686 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5687 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5688 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5689 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5690 }
5691}
5692
5693
5694/**
5695 * Determines if an exception is a contributory exception.
5696 *
5697 * Contributory exceptions are ones which can cause double-faults unless the
5698 * original exception was a benign exception. Page-fault is intentionally not
5699 * included here as it's a conditional contributory exception.
5700 *
5701 * @returns true if the exception is contributory, false otherwise.
5702 * @param uVector The exception vector.
5703 */
5704DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5705{
5706 switch (uVector)
5707 {
5708 case X86_XCPT_GP:
5709 case X86_XCPT_SS:
5710 case X86_XCPT_NP:
5711 case X86_XCPT_TS:
5712 case X86_XCPT_DE:
5713 return true;
5714 default:
5715 break;
5716 }
5717 return false;
5718}
5719
5720
5721/**
5722 * Sets an event as a pending event to be injected into the guest.
5723 *
5724 * @param pVCpu Pointer to the VMCPU.
5725 * @param u32IntInfo The VM-entry interruption-information field.
5726 * @param cbInstr The VM-entry instruction length in bytes (for software
5727 * interrupts, exceptions and privileged software
5728 * exceptions).
5729 * @param u32ErrCode The VM-entry exception error code.
5730 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5731 * page-fault.
5732 *
5733 * @remarks Statistics counter assumes this is a guest event being injected or
5734 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5735 * always incremented.
5736 */
5737DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5738 RTGCUINTPTR GCPtrFaultAddress)
5739{
5740 Assert(!pVCpu->hm.s.Event.fPending);
5741 pVCpu->hm.s.Event.fPending = true;
5742 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5743 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5744 pVCpu->hm.s.Event.cbInstr = cbInstr;
5745 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5746
5747 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5748}
5749
5750
5751/**
5752 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5753 *
5754 * @param pVCpu Pointer to the VMCPU.
5755 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5756 * out-of-sync. Make sure to update the required fields
5757 * before using them.
5758 */
5759DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5760{
5761 NOREF(pMixedCtx);
5762 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5763 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5764 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5765 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5766}
5767
5768
5769/**
5770 * Handle a condition that occurred while delivering an event through the guest
5771 * IDT.
5772 *
5773 * @returns VBox status code (informational error codes included).
5774 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5775 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5776 * continue execution of the guest which will delivery the #DF.
5777 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5778 *
5779 * @param pVCpu Pointer to the VMCPU.
5780 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5781 * out-of-sync. Make sure to update the required fields
5782 * before using them.
5783 * @param pVmxTransient Pointer to the VMX transient structure.
5784 *
5785 * @remarks No-long-jump zone!!!
5786 */
5787static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5788{
5789 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5790
5791 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5792 AssertRCReturn(rc, rc);
5793 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5794 AssertRCReturn(rc, rc);
5795
5796 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5797 {
5798 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5799 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5800
5801 typedef enum
5802 {
5803 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5804 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5805 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5806 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5807 } VMXREFLECTXCPT;
5808
5809 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5810 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5811 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5812 {
5813 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5814 {
5815 enmReflect = VMXREFLECTXCPT_XCPT;
5816#ifdef VBOX_STRICT
5817 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5818 && uExitVector == X86_XCPT_PF)
5819 {
5820 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5821 }
5822#endif
5823 if ( uExitVector == X86_XCPT_PF
5824 && uIdtVector == X86_XCPT_PF)
5825 {
5826 pVmxTransient->fVectoringDoublePF = true;
5827 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5828 }
5829 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5830 && hmR0VmxIsContributoryXcpt(uExitVector)
5831 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5832 || uIdtVector == X86_XCPT_PF))
5833 {
5834 enmReflect = VMXREFLECTXCPT_DF;
5835 }
5836 else if (uIdtVector == X86_XCPT_DF)
5837 enmReflect = VMXREFLECTXCPT_TF;
5838 }
5839 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5840 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5841 {
5842 /*
5843 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5844 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5845 */
5846 enmReflect = VMXREFLECTXCPT_XCPT;
5847
5848 if (uExitVector == X86_XCPT_PF)
5849 {
5850 pVmxTransient->fVectoringPF = true;
5851 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5852 }
5853 }
5854 }
5855 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5856 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5857 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5858 {
5859 /*
5860 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5861 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5862 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5863 */
5864 enmReflect = VMXREFLECTXCPT_XCPT;
5865 }
5866
5867 /*
5868 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5869 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5870 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5871 *
5872 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5873 */
5874 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5875 && enmReflect == VMXREFLECTXCPT_XCPT
5876 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5877 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5878 {
5879 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5880 }
5881
5882 switch (enmReflect)
5883 {
5884 case VMXREFLECTXCPT_XCPT:
5885 {
5886 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5887 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5888 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5889
5890 uint32_t u32ErrCode = 0;
5891 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5892 {
5893 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5894 AssertRCReturn(rc, rc);
5895 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5896 }
5897
5898 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5899 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5900 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5901 rc = VINF_SUCCESS;
5902 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5903 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5904
5905 break;
5906 }
5907
5908 case VMXREFLECTXCPT_DF:
5909 {
5910 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5911 rc = VINF_HM_DOUBLE_FAULT;
5912 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5913 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5914
5915 break;
5916 }
5917
5918 case VMXREFLECTXCPT_TF:
5919 {
5920 rc = VINF_EM_RESET;
5921 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5922 uExitVector));
5923 break;
5924 }
5925
5926 default:
5927 Assert(rc == VINF_SUCCESS);
5928 break;
5929 }
5930 }
5931 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5932 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5933 && uExitVector != X86_XCPT_DF
5934 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5935 {
5936 /*
5937 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5938 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5939 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5940 */
5941 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5942 {
5943 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5944 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5945 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5946 }
5947 }
5948
5949 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5950 return rc;
5951}
5952
5953
5954/**
5955 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5956 *
5957 * @returns VBox status code.
5958 * @param pVCpu Pointer to the VMCPU.
5959 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5960 * out-of-sync. Make sure to update the required fields
5961 * before using them.
5962 *
5963 * @remarks No-long-jump zone!!!
5964 */
5965static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5966{
5967 NOREF(pMixedCtx);
5968
5969 /*
5970 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5971 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5972 */
5973 VMMRZCallRing3Disable(pVCpu);
5974 HM_DISABLE_PREEMPT_IF_NEEDED();
5975
5976 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5977 {
5978 uint32_t uVal = 0;
5979 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5980 AssertRCReturn(rc, rc);
5981
5982 uint32_t uShadow = 0;
5983 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5984 AssertRCReturn(rc, rc);
5985
5986 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5987 CPUMSetGuestCR0(pVCpu, uVal);
5988 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5989 }
5990
5991 HM_RESTORE_PREEMPT_IF_NEEDED();
5992 VMMRZCallRing3Enable(pVCpu);
5993 return VINF_SUCCESS;
5994}
5995
5996
5997/**
5998 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5999 *
6000 * @returns VBox status code.
6001 * @param pVCpu Pointer to the VMCPU.
6002 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6003 * out-of-sync. Make sure to update the required fields
6004 * before using them.
6005 *
6006 * @remarks No-long-jump zone!!!
6007 */
6008static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6009{
6010 NOREF(pMixedCtx);
6011
6012 int rc = VINF_SUCCESS;
6013 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6014 {
6015 uint32_t uVal = 0;
6016 uint32_t uShadow = 0;
6017 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6018 AssertRCReturn(rc, rc);
6019 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6020 AssertRCReturn(rc, rc);
6021
6022 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6023 CPUMSetGuestCR4(pVCpu, uVal);
6024 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6025 }
6026 return rc;
6027}
6028
6029
6030/**
6031 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6032 *
6033 * @returns VBox status code.
6034 * @param pVCpu Pointer to the VMCPU.
6035 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6036 * out-of-sync. Make sure to update the required fields
6037 * before using them.
6038 *
6039 * @remarks No-long-jump zone!!!
6040 */
6041static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6042{
6043 int rc = VINF_SUCCESS;
6044 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6045 {
6046 uint64_t u64Val = 0;
6047 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6048 AssertRCReturn(rc, rc);
6049
6050 pMixedCtx->rip = u64Val;
6051 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6052 }
6053 return rc;
6054}
6055
6056
6057/**
6058 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6059 *
6060 * @returns VBox status code.
6061 * @param pVCpu Pointer to the VMCPU.
6062 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6063 * out-of-sync. Make sure to update the required fields
6064 * before using them.
6065 *
6066 * @remarks No-long-jump zone!!!
6067 */
6068static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6069{
6070 int rc = VINF_SUCCESS;
6071 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6072 {
6073 uint64_t u64Val = 0;
6074 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6075 AssertRCReturn(rc, rc);
6076
6077 pMixedCtx->rsp = u64Val;
6078 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6079 }
6080 return rc;
6081}
6082
6083
6084/**
6085 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6086 *
6087 * @returns VBox status code.
6088 * @param pVCpu Pointer to the VMCPU.
6089 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6090 * out-of-sync. Make sure to update the required fields
6091 * before using them.
6092 *
6093 * @remarks No-long-jump zone!!!
6094 */
6095static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6096{
6097 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6098 {
6099 uint32_t uVal = 0;
6100 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6101 AssertRCReturn(rc, rc);
6102
6103 pMixedCtx->eflags.u32 = uVal;
6104 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6105 {
6106 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6107 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6108
6109 pMixedCtx->eflags.Bits.u1VM = 0;
6110 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6111 }
6112
6113 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6114 }
6115 return VINF_SUCCESS;
6116}
6117
6118
6119/**
6120 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6121 * guest-CPU context.
6122 */
6123DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6124{
6125 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6126 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6127 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6128 return rc;
6129}
6130
6131
6132/**
6133 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6134 * from the guest-state area in the VMCS.
6135 *
6136 * @param pVCpu Pointer to the VMCPU.
6137 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6138 * out-of-sync. Make sure to update the required fields
6139 * before using them.
6140 *
6141 * @remarks No-long-jump zone!!!
6142 */
6143static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6144{
6145 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6146 {
6147 uint32_t uIntrState = 0;
6148 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6149 AssertRC(rc);
6150
6151 if (!uIntrState)
6152 {
6153 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6154 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6155
6156 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6157 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6158 }
6159 else
6160 {
6161 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6162 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6163 {
6164 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6165 AssertRC(rc);
6166 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6167 AssertRC(rc);
6168
6169 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6170 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6171 }
6172 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6173 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6174
6175 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6176 {
6177 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6178 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6179 }
6180 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6181 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6182 }
6183
6184 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6185 }
6186}
6187
6188
6189/**
6190 * Saves the guest's activity state.
6191 *
6192 * @returns VBox status code.
6193 * @param pVCpu Pointer to the VMCPU.
6194 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6195 * out-of-sync. Make sure to update the required fields
6196 * before using them.
6197 *
6198 * @remarks No-long-jump zone!!!
6199 */
6200static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6201{
6202 NOREF(pMixedCtx);
6203 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6204 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6205 return VINF_SUCCESS;
6206}
6207
6208
6209/**
6210 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6211 * the current VMCS into the guest-CPU context.
6212 *
6213 * @returns VBox status code.
6214 * @param pVCpu Pointer to the VMCPU.
6215 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6216 * out-of-sync. Make sure to update the required fields
6217 * before using them.
6218 *
6219 * @remarks No-long-jump zone!!!
6220 */
6221static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6222{
6223 int rc = VINF_SUCCESS;
6224 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6225 {
6226 uint32_t u32Val = 0;
6227 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6228 pMixedCtx->SysEnter.cs = u32Val;
6229 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6230 }
6231
6232 uint64_t u64Val = 0;
6233 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6234 {
6235 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6236 pMixedCtx->SysEnter.eip = u64Val;
6237 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6238 }
6239 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6240 {
6241 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6242 pMixedCtx->SysEnter.esp = u64Val;
6243 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6244 }
6245 return rc;
6246}
6247
6248
6249/**
6250 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6251 * the CPU back into the guest-CPU context.
6252 *
6253 * @returns VBox status code.
6254 * @param pVCpu Pointer to the VMCPU.
6255 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6256 * out-of-sync. Make sure to update the required fields
6257 * before using them.
6258 *
6259 * @remarks No-long-jump zone!!!
6260 */
6261static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6262{
6263#if HC_ARCH_BITS == 64
6264 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6265 {
6266 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6267 VMMRZCallRing3Disable(pVCpu);
6268 HM_DISABLE_PREEMPT_IF_NEEDED();
6269
6270 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6271 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6272 {
6273 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6274 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6275 }
6276
6277 HM_RESTORE_PREEMPT_IF_NEEDED();
6278 VMMRZCallRing3Enable(pVCpu);
6279 }
6280 else
6281 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6282#else
6283 NOREF(pMixedCtx);
6284 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6285#endif
6286
6287 return VINF_SUCCESS;
6288}
6289
6290
6291/**
6292 * Saves the auto load/store'd guest MSRs from the current VMCS into
6293 * the guest-CPU context.
6294 *
6295 * @returns VBox status code.
6296 * @param pVCpu Pointer to the VMCPU.
6297 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6298 * out-of-sync. Make sure to update the required fields
6299 * before using them.
6300 *
6301 * @remarks No-long-jump zone!!!
6302 */
6303static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6304{
6305 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6306 return VINF_SUCCESS;
6307
6308 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6309 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6310 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6311 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6312 {
6313 switch (pMsr->u32Msr)
6314 {
6315 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6316 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6317 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6318 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6319 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6320 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6321 break;
6322
6323 default:
6324 {
6325 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6326 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6327 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6328 }
6329 }
6330 }
6331
6332 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6333 return VINF_SUCCESS;
6334}
6335
6336
6337/**
6338 * Saves the guest control registers from the current VMCS into the guest-CPU
6339 * context.
6340 *
6341 * @returns VBox status code.
6342 * @param pVCpu Pointer to the VMCPU.
6343 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6344 * out-of-sync. Make sure to update the required fields
6345 * before using them.
6346 *
6347 * @remarks No-long-jump zone!!!
6348 */
6349static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6350{
6351 /* Guest CR0. Guest FPU. */
6352 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6353 AssertRCReturn(rc, rc);
6354
6355 /* Guest CR4. */
6356 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6357 AssertRCReturn(rc, rc);
6358
6359 /* Guest CR2 - updated always during the world-switch or in #PF. */
6360 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6361 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6362 {
6363 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6364 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6365
6366 PVM pVM = pVCpu->CTX_SUFF(pVM);
6367 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6368 || ( pVM->hm.s.fNestedPaging
6369 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6370 {
6371 uint64_t u64Val = 0;
6372 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6373 if (pMixedCtx->cr3 != u64Val)
6374 {
6375 CPUMSetGuestCR3(pVCpu, u64Val);
6376 if (VMMRZCallRing3IsEnabled(pVCpu))
6377 {
6378 PGMUpdateCR3(pVCpu, u64Val);
6379 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6380 }
6381 else
6382 {
6383 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6384 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6385 }
6386 }
6387
6388 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6389 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6390 {
6391 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6392 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6393 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6394 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6395
6396 if (VMMRZCallRing3IsEnabled(pVCpu))
6397 {
6398 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6399 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6400 }
6401 else
6402 {
6403 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6404 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6405 }
6406 }
6407 }
6408
6409 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6410 }
6411
6412 /*
6413 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6414 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6415 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6416 *
6417 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6418 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6419 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6420 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6421 *
6422 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6423 */
6424 if (VMMRZCallRing3IsEnabled(pVCpu))
6425 {
6426 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6427 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6428
6429 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6430 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6431
6432 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6433 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6434 }
6435
6436 return rc;
6437}
6438
6439
6440/**
6441 * Reads a guest segment register from the current VMCS into the guest-CPU
6442 * context.
6443 *
6444 * @returns VBox status code.
6445 * @param pVCpu Pointer to the VMCPU.
6446 * @param idxSel Index of the selector in the VMCS.
6447 * @param idxLimit Index of the segment limit in the VMCS.
6448 * @param idxBase Index of the segment base in the VMCS.
6449 * @param idxAccess Index of the access rights of the segment in the VMCS.
6450 * @param pSelReg Pointer to the segment selector.
6451 *
6452 * @remarks No-long-jump zone!!!
6453 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6454 * macro as that takes care of whether to read from the VMCS cache or
6455 * not.
6456 */
6457DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6458 PCPUMSELREG pSelReg)
6459{
6460 NOREF(pVCpu);
6461
6462 uint32_t u32Val = 0;
6463 int rc = VMXReadVmcs32(idxSel, &u32Val);
6464 AssertRCReturn(rc, rc);
6465 pSelReg->Sel = (uint16_t)u32Val;
6466 pSelReg->ValidSel = (uint16_t)u32Val;
6467 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6468
6469 rc = VMXReadVmcs32(idxLimit, &u32Val);
6470 AssertRCReturn(rc, rc);
6471 pSelReg->u32Limit = u32Val;
6472
6473 uint64_t u64Val = 0;
6474 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6475 AssertRCReturn(rc, rc);
6476 pSelReg->u64Base = u64Val;
6477
6478 rc = VMXReadVmcs32(idxAccess, &u32Val);
6479 AssertRCReturn(rc, rc);
6480 pSelReg->Attr.u = u32Val;
6481
6482 /*
6483 * If VT-x marks the segment as unusable, most other bits remain undefined:
6484 * - For CS the L, D and G bits have meaning.
6485 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6486 * - For the remaining data segments no bits are defined.
6487 *
6488 * The present bit and the unusable bit has been observed to be set at the
6489 * same time (the selector was supposed to be invalid as we started executing
6490 * a V8086 interrupt in ring-0).
6491 *
6492 * What should be important for the rest of the VBox code, is that the P bit is
6493 * cleared. Some of the other VBox code recognizes the unusable bit, but
6494 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6495 * safe side here, we'll strip off P and other bits we don't care about. If
6496 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6497 *
6498 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6499 */
6500 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6501 {
6502 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6503
6504 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6505 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6506 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6507
6508 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6509#ifdef DEBUG_bird
6510 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6511 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6512 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6513#endif
6514 }
6515 return VINF_SUCCESS;
6516}
6517
6518
6519#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6520# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6521 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6522 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6523#else
6524# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6525 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6526 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6527#endif
6528
6529
6530/**
6531 * Saves the guest segment registers from the current VMCS into the guest-CPU
6532 * context.
6533 *
6534 * @returns VBox status code.
6535 * @param pVCpu Pointer to the VMCPU.
6536 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6537 * out-of-sync. Make sure to update the required fields
6538 * before using them.
6539 *
6540 * @remarks No-long-jump zone!!!
6541 */
6542static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6543{
6544 /* Guest segment registers. */
6545 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6546 {
6547 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6548 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6549 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6550 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6551 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6552 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6553 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6554
6555 /* Restore segment attributes for real-on-v86 mode hack. */
6556 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6557 {
6558 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6559 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6560 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6561 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6562 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6563 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6564 }
6565 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6566 }
6567
6568 return VINF_SUCCESS;
6569}
6570
6571
6572/**
6573 * Saves the guest descriptor table registers and task register from the current
6574 * VMCS into the guest-CPU context.
6575 *
6576 * @returns VBox status code.
6577 * @param pVCpu Pointer to the VMCPU.
6578 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6579 * out-of-sync. Make sure to update the required fields
6580 * before using them.
6581 *
6582 * @remarks No-long-jump zone!!!
6583 */
6584static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6585{
6586 int rc = VINF_SUCCESS;
6587
6588 /* Guest LDTR. */
6589 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6590 {
6591 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6592 AssertRCReturn(rc, rc);
6593 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6594 }
6595
6596 /* Guest GDTR. */
6597 uint64_t u64Val = 0;
6598 uint32_t u32Val = 0;
6599 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6600 {
6601 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6602 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6603 pMixedCtx->gdtr.pGdt = u64Val;
6604 pMixedCtx->gdtr.cbGdt = u32Val;
6605 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6606 }
6607
6608 /* Guest IDTR. */
6609 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6610 {
6611 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6612 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6613 pMixedCtx->idtr.pIdt = u64Val;
6614 pMixedCtx->idtr.cbIdt = u32Val;
6615 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6616 }
6617
6618 /* Guest TR. */
6619 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6620 {
6621 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6622 AssertRCReturn(rc, rc);
6623
6624 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6625 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6626 {
6627 rc = VMXLOCAL_READ_SEG(TR, tr);
6628 AssertRCReturn(rc, rc);
6629 }
6630 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6631 }
6632 return rc;
6633}
6634
6635#undef VMXLOCAL_READ_SEG
6636
6637
6638/**
6639 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6640 * context.
6641 *
6642 * @returns VBox status code.
6643 * @param pVCpu Pointer to the VMCPU.
6644 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6645 * out-of-sync. Make sure to update the required fields
6646 * before using them.
6647 *
6648 * @remarks No-long-jump zone!!!
6649 */
6650static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6651{
6652 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6653 {
6654 if (!pVCpu->hm.s.fUsingHyperDR7)
6655 {
6656 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6657 uint32_t u32Val;
6658 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6659 pMixedCtx->dr[7] = u32Val;
6660 }
6661
6662 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6663 }
6664 return VINF_SUCCESS;
6665}
6666
6667
6668/**
6669 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6670 *
6671 * @returns VBox status code.
6672 * @param pVCpu Pointer to the VMCPU.
6673 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6674 * out-of-sync. Make sure to update the required fields
6675 * before using them.
6676 *
6677 * @remarks No-long-jump zone!!!
6678 */
6679static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6680{
6681 NOREF(pMixedCtx);
6682
6683 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6684 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6685 return VINF_SUCCESS;
6686}
6687
6688
6689/**
6690 * Saves the entire guest state from the currently active VMCS into the
6691 * guest-CPU context. This essentially VMREADs all guest-data.
6692 *
6693 * @returns VBox status code.
6694 * @param pVCpu Pointer to the VMCPU.
6695 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6696 * out-of-sync. Make sure to update the required fields
6697 * before using them.
6698 */
6699static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6700{
6701 Assert(pVCpu);
6702 Assert(pMixedCtx);
6703
6704 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6705 return VINF_SUCCESS;
6706
6707 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6708 again on the ring-3 callback path, there is no real need to. */
6709 if (VMMRZCallRing3IsEnabled(pVCpu))
6710 VMMR0LogFlushDisable(pVCpu);
6711 else
6712 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6713 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6714
6715 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6716 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6717
6718 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6719 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6720
6721 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6722 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6723
6724 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6725 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6726
6727 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6728 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6729
6730 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6731 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6732
6733 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6734 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6735
6736 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6737 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6738
6739 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6740 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6741
6742 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6743 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6744
6745 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6746 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6747
6748 if (VMMRZCallRing3IsEnabled(pVCpu))
6749 VMMR0LogFlushEnable(pVCpu);
6750
6751 return rc;
6752}
6753
6754
6755/**
6756 * Check per-VM and per-VCPU force flag actions that require us to go back to
6757 * ring-3 for one reason or another.
6758 *
6759 * @returns VBox status code (information status code included).
6760 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6761 * ring-3.
6762 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6763 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6764 * interrupts)
6765 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6766 * all EMTs to be in ring-3.
6767 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6768 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6769 * to the EM loop.
6770 *
6771 * @param pVM Pointer to the VM.
6772 * @param pVCpu Pointer to the VMCPU.
6773 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6774 * out-of-sync. Make sure to update the required fields
6775 * before using them.
6776 */
6777static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6778{
6779 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6780
6781 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6782 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6783 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6784 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6785 {
6786 /* We need the control registers now, make sure the guest-CPU context is updated. */
6787 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6788 AssertRCReturn(rc3, rc3);
6789
6790 /* Pending HM CR3 sync. */
6791 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6792 {
6793 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6794 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6795 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6796 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6797 }
6798
6799 /* Pending HM PAE PDPEs. */
6800 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6801 {
6802 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6803 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6804 }
6805
6806 /* Pending PGM C3 sync. */
6807 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6808 {
6809 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6810 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6811 if (rc2 != VINF_SUCCESS)
6812 {
6813 AssertRC(rc2);
6814 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6815 return rc2;
6816 }
6817 }
6818
6819 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6820 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6821 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6822 {
6823 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6824 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6825 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6826 return rc2;
6827 }
6828
6829 /* Pending VM request packets, such as hardware interrupts. */
6830 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6831 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6832 {
6833 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6834 return VINF_EM_PENDING_REQUEST;
6835 }
6836
6837 /* Pending PGM pool flushes. */
6838 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6839 {
6840 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6841 return VINF_PGM_POOL_FLUSH_PENDING;
6842 }
6843
6844 /* Pending DMA requests. */
6845 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6846 {
6847 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6848 return VINF_EM_RAW_TO_R3;
6849 }
6850 }
6851
6852 return VINF_SUCCESS;
6853}
6854
6855
6856/**
6857 * Converts any TRPM trap into a pending HM event. This is typically used when
6858 * entering from ring-3 (not longjmp returns).
6859 *
6860 * @param pVCpu Pointer to the VMCPU.
6861 */
6862static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6863{
6864 Assert(TRPMHasTrap(pVCpu));
6865 Assert(!pVCpu->hm.s.Event.fPending);
6866
6867 uint8_t uVector;
6868 TRPMEVENT enmTrpmEvent;
6869 RTGCUINT uErrCode;
6870 RTGCUINTPTR GCPtrFaultAddress;
6871 uint8_t cbInstr;
6872
6873 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6874 AssertRC(rc);
6875
6876 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6877 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6878 if (enmTrpmEvent == TRPM_TRAP)
6879 {
6880 switch (uVector)
6881 {
6882 case X86_XCPT_NMI:
6883 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6884 break;
6885
6886 case X86_XCPT_BP:
6887 case X86_XCPT_OF:
6888 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6889 break;
6890
6891 case X86_XCPT_PF:
6892 case X86_XCPT_DF:
6893 case X86_XCPT_TS:
6894 case X86_XCPT_NP:
6895 case X86_XCPT_SS:
6896 case X86_XCPT_GP:
6897 case X86_XCPT_AC:
6898 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6899 /* no break! */
6900 default:
6901 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6902 break;
6903 }
6904 }
6905 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6906 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6907 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6908 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6909 else
6910 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6911
6912 rc = TRPMResetTrap(pVCpu);
6913 AssertRC(rc);
6914 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6915 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6916
6917 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6918 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6919}
6920
6921
6922/**
6923 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6924 * VT-x to execute any instruction.
6925 *
6926 * @param pvCpu Pointer to the VMCPU.
6927 */
6928static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6929{
6930 Assert(pVCpu->hm.s.Event.fPending);
6931
6932 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6933 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6934 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6935 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6936
6937 /* If a trap was already pending, we did something wrong! */
6938 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6939
6940 TRPMEVENT enmTrapType;
6941 switch (uVectorType)
6942 {
6943 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6944 enmTrapType = TRPM_HARDWARE_INT;
6945 break;
6946
6947 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6948 enmTrapType = TRPM_SOFTWARE_INT;
6949 break;
6950
6951 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6952 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6953 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6954 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6955 enmTrapType = TRPM_TRAP;
6956 break;
6957
6958 default:
6959 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6960 enmTrapType = TRPM_32BIT_HACK;
6961 break;
6962 }
6963
6964 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6965
6966 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6967 AssertRC(rc);
6968
6969 if (fErrorCodeValid)
6970 TRPMSetErrorCode(pVCpu, uErrorCode);
6971
6972 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6973 && uVector == X86_XCPT_PF)
6974 {
6975 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6976 }
6977 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6978 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6979 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6980 {
6981 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6982 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6983 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6984 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6985 }
6986 pVCpu->hm.s.Event.fPending = false;
6987}
6988
6989
6990/**
6991 * Does the necessary state syncing before returning to ring-3 for any reason
6992 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6993 *
6994 * @returns VBox status code.
6995 * @param pVM Pointer to the VM.
6996 * @param pVCpu Pointer to the VMCPU.
6997 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6998 * be out-of-sync. Make sure to update the required
6999 * fields before using them.
7000 * @param fSaveGuestState Whether to save the guest state or not.
7001 *
7002 * @remarks No-long-jmp zone!!!
7003 */
7004static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7005{
7006 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7007 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7008
7009 RTCPUID idCpu = RTMpCpuId();
7010 Log4Func(("HostCpuId=%u\n", idCpu));
7011
7012 /*
7013 * !!! IMPORTANT !!!
7014 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7015 */
7016
7017 /* Save the guest state if necessary. */
7018 if ( fSaveGuestState
7019 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7020 {
7021 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7022 AssertRCReturn(rc, rc);
7023 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7024 }
7025
7026 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7027 if (CPUMIsGuestFPUStateActive(pVCpu))
7028 {
7029 /* We shouldn't reload CR0 without saving it first. */
7030 if (!fSaveGuestState)
7031 {
7032 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7033 AssertRCReturn(rc, rc);
7034 }
7035 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7036 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7037 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7038 }
7039
7040 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7041#ifdef VBOX_STRICT
7042 if (CPUMIsHyperDebugStateActive(pVCpu))
7043 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7044#endif
7045 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7046 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7047 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7048 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7049
7050#if HC_ARCH_BITS == 64
7051 /* Restore host-state bits that VT-x only restores partially. */
7052 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7053 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7054 {
7055 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7056 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7057 }
7058 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7059#endif
7060
7061#if HC_ARCH_BITS == 64
7062 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7063 if ( pVM->hm.s.fAllow64BitGuests
7064 && pVCpu->hm.s.vmx.fLazyMsrs)
7065 {
7066 /* We shouldn't reload the guest MSRs without saving it first. */
7067 if (!fSaveGuestState)
7068 {
7069 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7070 AssertRCReturn(rc, rc);
7071 }
7072 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7073 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7074 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7075 }
7076#endif
7077
7078 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7079 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7080
7081 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7082 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7083 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7084 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7085 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7086 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7087 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7088 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7089
7090 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7091
7092 /** @todo This partially defeats the purpose of having preemption hooks.
7093 * The problem is, deregistering the hooks should be moved to a place that
7094 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7095 * context.
7096 */
7097 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7098 {
7099 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7100 AssertRCReturn(rc, rc);
7101
7102 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7103 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7104 }
7105 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7106 NOREF(idCpu);
7107
7108 return VINF_SUCCESS;
7109}
7110
7111
7112/**
7113 * Leaves the VT-x session.
7114 *
7115 * @returns VBox status code.
7116 * @param pVM Pointer to the VM.
7117 * @param pVCpu Pointer to the VMCPU.
7118 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7119 * out-of-sync. Make sure to update the required fields
7120 * before using them.
7121 *
7122 * @remarks No-long-jmp zone!!!
7123 */
7124DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7125{
7126 HM_DISABLE_PREEMPT_IF_NEEDED();
7127 HMVMX_ASSERT_CPU_SAFE();
7128 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7129 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7130
7131 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7132 and done this from the VMXR0ThreadCtxCallback(). */
7133 if (!pVCpu->hm.s.fLeaveDone)
7134 {
7135 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7136 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7137 pVCpu->hm.s.fLeaveDone = true;
7138 }
7139 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7140
7141 /*
7142 * !!! IMPORTANT !!!
7143 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7144 */
7145
7146 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7147 /** @todo Deregistering here means we need to VMCLEAR always
7148 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7149 VMMR0ThreadCtxHooksDeregister(pVCpu);
7150
7151 /* Leave HM context. This takes care of local init (term). */
7152 int rc = HMR0LeaveCpu(pVCpu);
7153
7154 HM_RESTORE_PREEMPT_IF_NEEDED();
7155
7156 return rc;
7157}
7158
7159
7160/**
7161 * Does the necessary state syncing before doing a longjmp to ring-3.
7162 *
7163 * @returns VBox status code.
7164 * @param pVM Pointer to the VM.
7165 * @param pVCpu Pointer to the VMCPU.
7166 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7167 * out-of-sync. Make sure to update the required fields
7168 * before using them.
7169 *
7170 * @remarks No-long-jmp zone!!!
7171 */
7172DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7173{
7174 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7175}
7176
7177
7178/**
7179 * Take necessary actions before going back to ring-3.
7180 *
7181 * An action requires us to go back to ring-3. This function does the necessary
7182 * steps before we can safely return to ring-3. This is not the same as longjmps
7183 * to ring-3, this is voluntary and prepares the guest so it may continue
7184 * executing outside HM (recompiler/IEM).
7185 *
7186 * @returns VBox status code.
7187 * @param pVM Pointer to the VM.
7188 * @param pVCpu Pointer to the VMCPU.
7189 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7190 * out-of-sync. Make sure to update the required fields
7191 * before using them.
7192 * @param rcExit The reason for exiting to ring-3. Can be
7193 * VINF_VMM_UNKNOWN_RING3_CALL.
7194 */
7195static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7196{
7197 Assert(pVM);
7198 Assert(pVCpu);
7199 Assert(pMixedCtx);
7200 HMVMX_ASSERT_PREEMPT_SAFE();
7201
7202 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7203 {
7204 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7205 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7206 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7207 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7208 }
7209
7210 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7211 VMMRZCallRing3Disable(pVCpu);
7212 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7213
7214 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7215 if (pVCpu->hm.s.Event.fPending)
7216 {
7217 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7218 Assert(!pVCpu->hm.s.Event.fPending);
7219 }
7220
7221 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7222 and if we're injecting an event we should have a TRPM trap pending. */
7223 Assert(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu));
7224 Assert(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu));
7225
7226 /* Save guest state and restore host state bits. */
7227 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7228 AssertRCReturn(rc, rc);
7229 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7230 /* Thread-context hooks are unregistered at this point!!! */
7231
7232 /* Sync recompiler state. */
7233 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7234 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7235 | CPUM_CHANGED_LDTR
7236 | CPUM_CHANGED_GDTR
7237 | CPUM_CHANGED_IDTR
7238 | CPUM_CHANGED_TR
7239 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7240 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7241 if ( pVM->hm.s.fNestedPaging
7242 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7243 {
7244 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7245 }
7246
7247 Assert(!pVCpu->hm.s.fClearTrapFlag);
7248
7249 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7250 if (rcExit != VINF_EM_RAW_INTERRUPT)
7251 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7252
7253 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7254
7255 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7256 VMMRZCallRing3RemoveNotification(pVCpu);
7257 VMMRZCallRing3Enable(pVCpu);
7258
7259 return rc;
7260}
7261
7262
7263/**
7264 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7265 * longjump to ring-3 and possibly get preempted.
7266 *
7267 * @returns VBox status code.
7268 * @param pVCpu Pointer to the VMCPU.
7269 * @param enmOperation The operation causing the ring-3 longjump.
7270 * @param pvUser Opaque pointer to the guest-CPU context. The data
7271 * may be out-of-sync. Make sure to update the required
7272 * fields before using them.
7273 */
7274DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7275{
7276 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7277 {
7278 /*
7279 * !!! IMPORTANT !!!
7280 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7281 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7282 */
7283 VMMRZCallRing3RemoveNotification(pVCpu);
7284 VMMRZCallRing3Disable(pVCpu);
7285 HM_DISABLE_PREEMPT_IF_NEEDED();
7286
7287 PVM pVM = pVCpu->CTX_SUFF(pVM);
7288 if (CPUMIsGuestFPUStateActive(pVCpu))
7289 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7290
7291 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7292
7293#if HC_ARCH_BITS == 64
7294 /* Restore host-state bits that VT-x only restores partially. */
7295 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7296 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7297 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7298 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7299
7300 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7301 if ( pVM->hm.s.fAllow64BitGuests
7302 && pVCpu->hm.s.vmx.fLazyMsrs)
7303 {
7304 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7305 }
7306#endif
7307 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7308 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7309 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7310 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7311 {
7312 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7313 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7314 }
7315
7316 VMMR0ThreadCtxHooksDeregister(pVCpu);
7317
7318 HMR0LeaveCpu(pVCpu);
7319 HM_RESTORE_PREEMPT_IF_NEEDED();
7320 return VINF_SUCCESS;
7321 }
7322
7323 Assert(pVCpu);
7324 Assert(pvUser);
7325 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7326 HMVMX_ASSERT_PREEMPT_SAFE();
7327
7328 VMMRZCallRing3Disable(pVCpu);
7329 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7330
7331 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7332 enmOperation));
7333
7334 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7335 AssertRCReturn(rc, rc);
7336
7337 VMMRZCallRing3Enable(pVCpu);
7338 return VINF_SUCCESS;
7339}
7340
7341
7342/**
7343 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7344 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7345 *
7346 * @param pVCpu Pointer to the VMCPU.
7347 */
7348DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7349{
7350 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7351 {
7352 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7353 {
7354 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7355 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7356 AssertRC(rc);
7357 Log4(("Setup interrupt-window exiting\n"));
7358 }
7359 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7360}
7361
7362
7363/**
7364 * Clears the interrupt-window exiting control in the VMCS.
7365 *
7366 * @param pVCpu Pointer to the VMCPU.
7367 */
7368DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7369{
7370 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7371 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7372 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7373 AssertRC(rc);
7374 Log4(("Cleared interrupt-window exiting\n"));
7375}
7376
7377
7378/**
7379 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7380 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7381 *
7382 * @param pVCpu Pointer to the VMCPU.
7383 */
7384DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7385{
7386 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7387 {
7388 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7389 {
7390 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7391 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7392 AssertRC(rc);
7393 Log4(("Setup NMI-window exiting\n"));
7394 }
7395 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7396}
7397
7398
7399/**
7400 * Clears the NMI-window exiting control in the VMCS.
7401 *
7402 * @param pVCpu Pointer to the VMCPU.
7403 */
7404DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7405{
7406 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7407 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7408 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7409 AssertRC(rc);
7410 Log4(("Cleared NMI-window exiting\n"));
7411}
7412
7413
7414/**
7415 * Evaluates the event to be delivered to the guest and sets it as the pending
7416 * event.
7417 *
7418 * @param pVCpu Pointer to the VMCPU.
7419 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7420 * out-of-sync. Make sure to update the required fields
7421 * before using them.
7422 */
7423static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7424{
7425 Assert(!pVCpu->hm.s.Event.fPending);
7426
7427 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7428 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7429 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7430 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7431 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7432
7433 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7434 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7435 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7436 Assert(!TRPMHasTrap(pVCpu));
7437
7438 /*
7439 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7440 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7441 */
7442 /** @todo SMI. SMIs take priority over NMIs. */
7443 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7444 {
7445 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7446 if ( !fBlockNmi
7447 && !fBlockSti
7448 && !fBlockMovSS)
7449 {
7450 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7451 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7452 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7453
7454 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7455 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7456 }
7457 else
7458 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7459 }
7460 /*
7461 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7462 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7463 */
7464 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7465 && !pVCpu->hm.s.fSingleInstruction)
7466 {
7467 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7468 AssertRC(rc);
7469 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7470 if ( !fBlockInt
7471 && !fBlockSti
7472 && !fBlockMovSS)
7473 {
7474 uint8_t u8Interrupt;
7475 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7476 if (RT_SUCCESS(rc))
7477 {
7478 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7479 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7480 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7481
7482 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7483 }
7484 else
7485 {
7486 /** @todo Does this actually happen? If not turn it into an assertion. */
7487 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7488 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7489 }
7490 }
7491 else
7492 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7493 }
7494}
7495
7496
7497/**
7498 * Sets a pending-debug exception to be delivered to the guest if the guest is
7499 * single-stepping.
7500 *
7501 * @param pVCpu Pointer to the VMCPU.
7502 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7503 * out-of-sync. Make sure to update the required fields
7504 * before using them.
7505 */
7506DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7507{
7508 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7509 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7510 {
7511 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7512 AssertRC(rc);
7513 }
7514}
7515
7516
7517/**
7518 * Injects any pending events into the guest if the guest is in a state to
7519 * receive them.
7520 *
7521 * @returns VBox status code (informational status codes included).
7522 * @param pVCpu Pointer to the VMCPU.
7523 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7524 * out-of-sync. Make sure to update the required fields
7525 * before using them.
7526 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7527 * return VINF_EM_DBG_STEPPED if the event was
7528 * dispatched directly.
7529 */
7530static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7531{
7532 HMVMX_ASSERT_PREEMPT_SAFE();
7533 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7534
7535 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7536 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7537 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7538 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7539
7540 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7541 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7542 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7543 Assert(!TRPMHasTrap(pVCpu));
7544
7545 int rc = VINF_SUCCESS;
7546 if (pVCpu->hm.s.Event.fPending)
7547 {
7548 /*
7549 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7550 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7551 * ended up enabling interrupts outside VT-x.
7552 */
7553 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7554 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7555 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7556 {
7557 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7558 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7559 }
7560
7561#ifdef VBOX_STRICT
7562 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7563 {
7564 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7565 Assert(!fBlockInt);
7566 Assert(!fBlockSti);
7567 Assert(!fBlockMovSS);
7568 }
7569 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7570 {
7571 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7572 Assert(!fBlockSti);
7573 Assert(!fBlockMovSS);
7574 Assert(!fBlockNmi);
7575 }
7576#endif
7577 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7578 (uint8_t)uIntType));
7579 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7580 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7581 AssertRCReturn(rc, rc);
7582
7583 /* Update the interruptibility-state as it could have been changed by
7584 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7585 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7586 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7587
7588#ifdef VBOX_WITH_STATISTICS
7589 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7590 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7591 else
7592 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7593#endif
7594 }
7595
7596 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7597 if ( fBlockSti
7598 || fBlockMovSS)
7599 {
7600 if ( !pVCpu->hm.s.fSingleInstruction
7601 && !DBGFIsStepping(pVCpu))
7602 {
7603 /*
7604 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7605 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7606 * See Intel spec. 27.3.4 "Saving Non-Register State".
7607 */
7608 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7609 AssertRCReturn(rc2, rc2);
7610 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7611 }
7612 else if (pMixedCtx->eflags.Bits.u1TF)
7613 {
7614 /*
7615 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7616 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7617 */
7618 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7619 uIntrState = 0;
7620 }
7621 }
7622
7623 /*
7624 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7625 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7626 */
7627 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7628 AssertRC(rc2);
7629
7630 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7631 NOREF(fBlockMovSS); NOREF(fBlockSti);
7632 return rc;
7633}
7634
7635
7636/**
7637 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7638 *
7639 * @param pVCpu Pointer to the VMCPU.
7640 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7641 * out-of-sync. Make sure to update the required fields
7642 * before using them.
7643 */
7644DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7645{
7646 NOREF(pMixedCtx);
7647 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7648 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7649}
7650
7651
7652/**
7653 * Injects a double-fault (#DF) exception into the VM.
7654 *
7655 * @returns VBox status code (informational status code included).
7656 * @param pVCpu Pointer to the VMCPU.
7657 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7658 * out-of-sync. Make sure to update the required fields
7659 * before using them.
7660 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7661 * and should return VINF_EM_DBG_STEPPED if the event
7662 * is injected directly (register modified by us, not
7663 * by hardware on VM-entry).
7664 * @param puIntrState Pointer to the current guest interruptibility-state.
7665 * This interruptibility-state will be updated if
7666 * necessary. This cannot not be NULL.
7667 */
7668DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7669{
7670 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7671 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7672 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7673 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7674 fStepping, puIntrState);
7675}
7676
7677
7678/**
7679 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7680 *
7681 * @param pVCpu Pointer to the VMCPU.
7682 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7683 * out-of-sync. Make sure to update the required fields
7684 * before using them.
7685 */
7686DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7687{
7688 NOREF(pMixedCtx);
7689 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7690 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7691 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7692}
7693
7694
7695/**
7696 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7697 *
7698 * @param pVCpu Pointer to the VMCPU.
7699 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7700 * out-of-sync. Make sure to update the required fields
7701 * before using them.
7702 * @param cbInstr The value of RIP that is to be pushed on the guest
7703 * stack.
7704 */
7705DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7706{
7707 NOREF(pMixedCtx);
7708 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7709 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7710 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7711}
7712
7713
7714/**
7715 * Injects a general-protection (#GP) fault into the VM.
7716 *
7717 * @returns VBox status code (informational status code included).
7718 * @param pVCpu Pointer to the VMCPU.
7719 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7720 * out-of-sync. Make sure to update the required fields
7721 * before using them.
7722 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7723 * mode, i.e. in real-mode it's not valid).
7724 * @param u32ErrorCode The error code associated with the #GP.
7725 * @param fStepping Whether we're running in
7726 * hmR0VmxRunGuestCodeStep() and should return
7727 * VINF_EM_DBG_STEPPED if the event is injected
7728 * directly (register modified by us, not by
7729 * hardware on VM-entry).
7730 * @param puIntrState Pointer to the current guest interruptibility-state.
7731 * This interruptibility-state will be updated if
7732 * necessary. This cannot not be NULL.
7733 */
7734DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7735 bool fStepping, uint32_t *puIntrState)
7736{
7737 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7738 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7739 if (fErrorCodeValid)
7740 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7741 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7742 fStepping, puIntrState);
7743}
7744
7745
7746/**
7747 * Sets a general-protection (#GP) exception as pending-for-injection into the
7748 * VM.
7749 *
7750 * @param pVCpu Pointer to the VMCPU.
7751 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7752 * out-of-sync. Make sure to update the required fields
7753 * before using them.
7754 * @param u32ErrorCode The error code associated with the #GP.
7755 */
7756DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7757{
7758 NOREF(pMixedCtx);
7759 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7760 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7761 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7762 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7763}
7764
7765
7766/**
7767 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7768 *
7769 * @param pVCpu Pointer to the VMCPU.
7770 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7771 * out-of-sync. Make sure to update the required fields
7772 * before using them.
7773 * @param uVector The software interrupt vector number.
7774 * @param cbInstr The value of RIP that is to be pushed on the guest
7775 * stack.
7776 */
7777DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7778{
7779 NOREF(pMixedCtx);
7780 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7781 if ( uVector == X86_XCPT_BP
7782 || uVector == X86_XCPT_OF)
7783 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7784 else
7785 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7786 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7787}
7788
7789
7790/**
7791 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7792 * stack.
7793 *
7794 * @returns VBox status code (information status code included).
7795 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7796 * @param pVM Pointer to the VM.
7797 * @param pMixedCtx Pointer to the guest-CPU context.
7798 * @param uValue The value to push to the guest stack.
7799 */
7800DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7801{
7802 /*
7803 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7804 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7805 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7806 */
7807 if (pMixedCtx->sp == 1)
7808 return VINF_EM_RESET;
7809 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7810 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7811 AssertRCReturn(rc, rc);
7812 return rc;
7813}
7814
7815
7816/**
7817 * Injects an event into the guest upon VM-entry by updating the relevant fields
7818 * in the VM-entry area in the VMCS.
7819 *
7820 * @returns VBox status code (informational error codes included).
7821 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7822 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7823 *
7824 * @param pVCpu Pointer to the VMCPU.
7825 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7826 * be out-of-sync. Make sure to update the required
7827 * fields before using them.
7828 * @param u64IntInfo The VM-entry interruption-information field.
7829 * @param cbInstr The VM-entry instruction length in bytes (for
7830 * software interrupts, exceptions and privileged
7831 * software exceptions).
7832 * @param u32ErrCode The VM-entry exception error code.
7833 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7834 * @param puIntrState Pointer to the current guest interruptibility-state.
7835 * This interruptibility-state will be updated if
7836 * necessary. This cannot not be NULL.
7837 * @param fStepping Whether we're running in
7838 * hmR0VmxRunGuestCodeStep() and should return
7839 * VINF_EM_DBG_STEPPED if the event is injected
7840 * directly (register modified by us, not by
7841 * hardware on VM-entry).
7842 *
7843 * @remarks Requires CR0!
7844 * @remarks No-long-jump zone!!!
7845 */
7846static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7847 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7848{
7849 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7850 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7851 Assert(puIntrState);
7852 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7853
7854 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7855 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7856
7857#ifdef VBOX_STRICT
7858 /* Validate the error-code-valid bit for hardware exceptions. */
7859 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7860 {
7861 switch (uVector)
7862 {
7863 case X86_XCPT_PF:
7864 case X86_XCPT_DF:
7865 case X86_XCPT_TS:
7866 case X86_XCPT_NP:
7867 case X86_XCPT_SS:
7868 case X86_XCPT_GP:
7869 case X86_XCPT_AC:
7870 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7871 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7872 /* fallthru */
7873 default:
7874 break;
7875 }
7876 }
7877#endif
7878
7879 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7880 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7881 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7882
7883 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7884
7885 /* We require CR0 to check if the guest is in real-mode. */
7886 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7887 AssertRCReturn(rc, rc);
7888
7889 /*
7890 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7891 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7892 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7893 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7894 */
7895 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7896 {
7897 PVM pVM = pVCpu->CTX_SUFF(pVM);
7898 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7899 {
7900 Assert(PDMVmmDevHeapIsEnabled(pVM));
7901 Assert(pVM->hm.s.vmx.pRealModeTSS);
7902
7903 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7904 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7905 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7906 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7907 AssertRCReturn(rc, rc);
7908 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7909
7910 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7911 size_t const cbIdtEntry = sizeof(X86IDTR16);
7912 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7913 {
7914 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7915 if (uVector == X86_XCPT_DF)
7916 return VINF_EM_RESET;
7917
7918 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7919 if (uVector == X86_XCPT_GP)
7920 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7921
7922 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7923 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7924 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7925 fStepping, puIntrState);
7926 }
7927
7928 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7929 uint16_t uGuestIp = pMixedCtx->ip;
7930 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7931 {
7932 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7933 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7934 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7935 }
7936 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7937 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7938
7939 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7940 X86IDTR16 IdtEntry;
7941 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7942 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7943 AssertRCReturn(rc, rc);
7944
7945 /* Construct the stack frame for the interrupt/exception handler. */
7946 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7947 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7948 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7949 AssertRCReturn(rc, rc);
7950
7951 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7952 if (rc == VINF_SUCCESS)
7953 {
7954 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7955 pMixedCtx->rip = IdtEntry.offSel;
7956 pMixedCtx->cs.Sel = IdtEntry.uSel;
7957 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7958 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7959 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7960 && uVector == X86_XCPT_PF)
7961 pMixedCtx->cr2 = GCPtrFaultAddress;
7962
7963 /* If any other guest-state bits are changed here, make sure to update
7964 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7965 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7966 | HM_CHANGED_GUEST_RIP
7967 | HM_CHANGED_GUEST_RFLAGS
7968 | HM_CHANGED_GUEST_RSP);
7969
7970 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7971 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7972 {
7973 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7974 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7975 Log4(("Clearing inhibition due to STI.\n"));
7976 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7977 }
7978 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7979 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7980
7981 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7982 it, if we are returning to ring-3 before executing guest code. */
7983 pVCpu->hm.s.Event.fPending = false;
7984
7985 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7986 if (fStepping)
7987 rc = VINF_EM_DBG_STEPPED;
7988 }
7989 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7990 return rc;
7991 }
7992
7993 /*
7994 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7995 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7996 */
7997 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7998 }
7999
8000 /* Validate. */
8001 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8002 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8003 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8004
8005 /* Inject. */
8006 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8007 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8008 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8009 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8010
8011 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8012 && uVector == X86_XCPT_PF)
8013 pMixedCtx->cr2 = GCPtrFaultAddress;
8014
8015 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8016 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8017
8018 AssertRCReturn(rc, rc);
8019 return rc;
8020}
8021
8022
8023/**
8024 * Clears the interrupt-window exiting control in the VMCS and if necessary
8025 * clears the current event in the VMCS as well.
8026 *
8027 * @returns VBox status code.
8028 * @param pVCpu Pointer to the VMCPU.
8029 *
8030 * @remarks Use this function only to clear events that have not yet been
8031 * delivered to the guest but are injected in the VMCS!
8032 * @remarks No-long-jump zone!!!
8033 */
8034static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
8035{
8036 int rc;
8037 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8038
8039 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8040 {
8041 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8042 Assert(!pVCpu->hm.s.Event.fPending);
8043 }
8044
8045 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8046 {
8047 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8048 Assert(!pVCpu->hm.s.Event.fPending);
8049 }
8050
8051 if (!pVCpu->hm.s.Event.fPending)
8052 return;
8053
8054#ifdef VBOX_STRICT
8055 uint32_t u32EntryInfo;
8056 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8057 AssertRC(rc);
8058 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8059#endif
8060
8061 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8062 AssertRC(rc);
8063
8064 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8065 AssertRC(rc);
8066
8067 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8068 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8069}
8070
8071
8072/**
8073 * Enters the VT-x session.
8074 *
8075 * @returns VBox status code.
8076 * @param pVM Pointer to the VM.
8077 * @param pVCpu Pointer to the VMCPU.
8078 * @param pCpu Pointer to the CPU info struct.
8079 */
8080VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8081{
8082 AssertPtr(pVM);
8083 AssertPtr(pVCpu);
8084 Assert(pVM->hm.s.vmx.fSupported);
8085 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8086 NOREF(pCpu); NOREF(pVM);
8087
8088 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8089 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8090
8091#ifdef VBOX_STRICT
8092 /* Make sure we're in VMX root mode. */
8093 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8094 if (!(u32HostCR4 & X86_CR4_VMXE))
8095 {
8096 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8097 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8098 }
8099#endif
8100
8101 /*
8102 * Load the VCPU's VMCS as the current (and active) one.
8103 */
8104 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8105 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8106 if (RT_FAILURE(rc))
8107 return rc;
8108
8109 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8110 pVCpu->hm.s.fLeaveDone = false;
8111 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8112
8113 return VINF_SUCCESS;
8114}
8115
8116
8117/**
8118 * The thread-context callback (only on platforms which support it).
8119 *
8120 * @param enmEvent The thread-context event.
8121 * @param pVCpu Pointer to the VMCPU.
8122 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8123 * @thread EMT(pVCpu)
8124 */
8125VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8126{
8127 NOREF(fGlobalInit);
8128
8129 switch (enmEvent)
8130 {
8131 case RTTHREADCTXEVENT_PREEMPTING:
8132 {
8133 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8134 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8135 VMCPU_ASSERT_EMT(pVCpu);
8136
8137 PVM pVM = pVCpu->CTX_SUFF(pVM);
8138 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8139
8140 /* No longjmps (logger flushes, locks) in this fragile context. */
8141 VMMRZCallRing3Disable(pVCpu);
8142 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8143
8144 /*
8145 * Restore host-state (FPU, debug etc.)
8146 */
8147 if (!pVCpu->hm.s.fLeaveDone)
8148 {
8149 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8150 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8151 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8152 pVCpu->hm.s.fLeaveDone = true;
8153 }
8154
8155 /* Leave HM context, takes care of local init (term). */
8156 int rc = HMR0LeaveCpu(pVCpu);
8157 AssertRC(rc); NOREF(rc);
8158
8159 /* Restore longjmp state. */
8160 VMMRZCallRing3Enable(pVCpu);
8161 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8162 break;
8163 }
8164
8165 case RTTHREADCTXEVENT_RESUMED:
8166 {
8167 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8168 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8169 VMCPU_ASSERT_EMT(pVCpu);
8170
8171 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8172 VMMRZCallRing3Disable(pVCpu);
8173 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8174
8175 /* Initialize the bare minimum state required for HM. This takes care of
8176 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8177 int rc = HMR0EnterCpu(pVCpu);
8178 AssertRC(rc);
8179 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8180
8181 /* Load the active VMCS as the current one. */
8182 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8183 {
8184 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8185 AssertRC(rc); NOREF(rc);
8186 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8187 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8188 }
8189 pVCpu->hm.s.fLeaveDone = false;
8190
8191 /* Restore longjmp state. */
8192 VMMRZCallRing3Enable(pVCpu);
8193 break;
8194 }
8195
8196 default:
8197 break;
8198 }
8199}
8200
8201
8202/**
8203 * Saves the host state in the VMCS host-state.
8204 * Sets up the VM-exit MSR-load area.
8205 *
8206 * The CPU state will be loaded from these fields on every successful VM-exit.
8207 *
8208 * @returns VBox status code.
8209 * @param pVM Pointer to the VM.
8210 * @param pVCpu Pointer to the VMCPU.
8211 *
8212 * @remarks No-long-jump zone!!!
8213 */
8214static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8215{
8216 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8217
8218 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8219 return VINF_SUCCESS;
8220
8221 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8222 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8223
8224 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8225 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8226
8227 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8228 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8229
8230 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8231 return rc;
8232}
8233
8234
8235/**
8236 * Saves the host state in the VMCS host-state.
8237 *
8238 * @returns VBox status code.
8239 * @param pVM Pointer to the VM.
8240 * @param pVCpu Pointer to the VMCPU.
8241 *
8242 * @remarks No-long-jump zone!!!
8243 */
8244VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8245{
8246 AssertPtr(pVM);
8247 AssertPtr(pVCpu);
8248
8249 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8250
8251 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8252 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8253 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8254 return hmR0VmxSaveHostState(pVM, pVCpu);
8255}
8256
8257
8258/**
8259 * Loads the guest state into the VMCS guest-state area.
8260 *
8261 * The will typically be done before VM-entry when the guest-CPU state and the
8262 * VMCS state may potentially be out of sync.
8263 *
8264 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8265 * VM-entry controls.
8266 * Sets up the appropriate VMX non-root function to execute guest code based on
8267 * the guest CPU mode.
8268 *
8269 * @returns VBox status code.
8270 * @param pVM Pointer to the VM.
8271 * @param pVCpu Pointer to the VMCPU.
8272 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8273 * out-of-sync. Make sure to update the required fields
8274 * before using them.
8275 *
8276 * @remarks No-long-jump zone!!!
8277 */
8278static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8279{
8280 AssertPtr(pVM);
8281 AssertPtr(pVCpu);
8282 AssertPtr(pMixedCtx);
8283 HMVMX_ASSERT_PREEMPT_SAFE();
8284
8285 VMMRZCallRing3Disable(pVCpu);
8286 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8287
8288 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8289
8290 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8291
8292 /* Determine real-on-v86 mode. */
8293 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8294 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8295 && CPUMIsGuestInRealModeEx(pMixedCtx))
8296 {
8297 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8298 }
8299
8300 /*
8301 * Load the guest-state into the VMCS.
8302 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8303 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8304 */
8305 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8306 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8307
8308 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8309 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8310 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8311
8312 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8313 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8314 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8315
8316 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8317 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8318
8319 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8320 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8321
8322 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8323 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8324 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8325
8326 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8327 determine we don't have to swap EFER after all. */
8328 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8329 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8330
8331 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8332 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8333
8334 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8335 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8336
8337 /*
8338 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8339 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8340 */
8341 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8342 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8343
8344 /* Clear any unused and reserved bits. */
8345 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8346
8347 VMMRZCallRing3Enable(pVCpu);
8348
8349 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8350 return rc;
8351}
8352
8353
8354/**
8355 * Loads the state shared between the host and guest into the VMCS.
8356 *
8357 * @param pVM Pointer to the VM.
8358 * @param pVCpu Pointer to the VMCPU.
8359 * @param pCtx Pointer to the guest-CPU context.
8360 *
8361 * @remarks No-long-jump zone!!!
8362 */
8363static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8364{
8365 NOREF(pVM);
8366
8367 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8368 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8369
8370 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8371 {
8372 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8373 AssertRC(rc);
8374 }
8375
8376 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8377 {
8378 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8379 AssertRC(rc);
8380
8381 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8382 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8383 {
8384 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8385 AssertRC(rc);
8386 }
8387 }
8388
8389 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8390 {
8391#if HC_ARCH_BITS == 64
8392 if (pVM->hm.s.fAllow64BitGuests)
8393 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8394#endif
8395 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8396 }
8397
8398 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8399 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8400 {
8401 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8402 AssertRC(rc);
8403 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8404 }
8405
8406 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8407 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8408}
8409
8410
8411/**
8412 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8413 *
8414 * @param pVM Pointer to the VM.
8415 * @param pVCpu Pointer to the VMCPU.
8416 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8417 * out-of-sync. Make sure to update the required fields
8418 * before using them.
8419 */
8420DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8421{
8422 HMVMX_ASSERT_PREEMPT_SAFE();
8423
8424 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8425#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8426 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8427#endif
8428
8429 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8430 {
8431 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8432 AssertRC(rc);
8433 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8434 }
8435 else if (HMCPU_CF_VALUE(pVCpu))
8436 {
8437 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8438 AssertRC(rc);
8439 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8440 }
8441
8442 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8443 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8444 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8445 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8446}
8447
8448
8449/**
8450 * Does the preparations before executing guest code in VT-x.
8451 *
8452 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8453 * recompiler/IEM. We must be cautious what we do here regarding committing
8454 * guest-state information into the VMCS assuming we assuredly execute the
8455 * guest in VT-x mode.
8456 *
8457 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8458 * the common-state (TRPM/forceflags), we must undo those changes so that the
8459 * recompiler/IEM can (and should) use them when it resumes guest execution.
8460 * Otherwise such operations must be done when we can no longer exit to ring-3.
8461 *
8462 * @returns Strict VBox status code.
8463 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8464 * have been disabled.
8465 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8466 * double-fault into the guest.
8467 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8468 * dispatched directly.
8469 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8470 *
8471 * @param pVM Pointer to the VM.
8472 * @param pVCpu Pointer to the VMCPU.
8473 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8474 * out-of-sync. Make sure to update the required fields
8475 * before using them.
8476 * @param pVmxTransient Pointer to the VMX transient structure.
8477 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8478 * us ignore some of the reasons for returning to
8479 * ring-3, and return VINF_EM_DBG_STEPPED if event
8480 * dispatching took place.
8481 */
8482static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8483{
8484 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8485
8486#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8487 PGMRZDynMapFlushAutoSet(pVCpu);
8488#endif
8489
8490 /* Check force flag actions that might require us to go back to ring-3. */
8491 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8492 if (rc != VINF_SUCCESS)
8493 return rc;
8494
8495#ifndef IEM_VERIFICATION_MODE_FULL
8496 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8497 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8498 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8499 {
8500 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8501 RTGCPHYS GCPhysApicBase;
8502 GCPhysApicBase = pMixedCtx->msrApicBase;
8503 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8504
8505 /* Unalias any existing mapping. */
8506 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8507 AssertRCReturn(rc, rc);
8508
8509 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8510 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8511 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8512 AssertRCReturn(rc, rc);
8513
8514 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8515 }
8516#endif /* !IEM_VERIFICATION_MODE_FULL */
8517
8518 if (TRPMHasTrap(pVCpu))
8519 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8520 else if (!pVCpu->hm.s.Event.fPending)
8521 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8522
8523 /*
8524 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8525 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8526 */
8527 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8528 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8529 {
8530 Assert(rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
8531 return rc;
8532 }
8533
8534 /*
8535 * Load the guest state bits, we can handle longjmps/getting preempted here.
8536 *
8537 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8538 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8539 * Hence, this needs to be done -after- injection of events.
8540 */
8541 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8542
8543 /*
8544 * No longjmps to ring-3 from this point on!!!
8545 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8546 * This also disables flushing of the R0-logger instance (if any).
8547 */
8548 VMMRZCallRing3Disable(pVCpu);
8549
8550 /*
8551 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8552 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8553 *
8554 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8555 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8556 *
8557 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8558 * executing guest code.
8559 */
8560 pVmxTransient->uEflags = ASMIntDisableFlags();
8561 if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8562 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8563 && ( !fStepping /* Optimized for the non-stepping case, of course. */
8564 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8565 {
8566 hmR0VmxClearEventVmcs(pVCpu);
8567 ASMSetFlags(pVmxTransient->uEflags);
8568 VMMRZCallRing3Enable(pVCpu);
8569 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8570 return VINF_EM_RAW_TO_R3;
8571 }
8572
8573 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8574 {
8575 hmR0VmxClearEventVmcs(pVCpu);
8576 ASMSetFlags(pVmxTransient->uEflags);
8577 VMMRZCallRing3Enable(pVCpu);
8578 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8579 return VINF_EM_RAW_INTERRUPT;
8580 }
8581
8582 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8583 pVCpu->hm.s.Event.fPending = false;
8584
8585 return VINF_SUCCESS;
8586}
8587
8588
8589/**
8590 * Prepares to run guest code in VT-x and we've committed to doing so. This
8591 * means there is no backing out to ring-3 or anywhere else at this
8592 * point.
8593 *
8594 * @param pVM Pointer to the VM.
8595 * @param pVCpu Pointer to the VMCPU.
8596 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8597 * out-of-sync. Make sure to update the required fields
8598 * before using them.
8599 * @param pVmxTransient Pointer to the VMX transient structure.
8600 *
8601 * @remarks Called with preemption disabled.
8602 * @remarks No-long-jump zone!!!
8603 */
8604static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8605{
8606 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8607 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8608 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8609
8610 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8611 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8612
8613#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8614 if (!CPUMIsGuestFPUStateActive(pVCpu))
8615 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8616 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8617#endif
8618
8619 if ( pVCpu->hm.s.fPreloadGuestFpu
8620 && !CPUMIsGuestFPUStateActive(pVCpu))
8621 {
8622 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8623 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8624 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8625 }
8626
8627 /*
8628 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8629 */
8630 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8631 && pVCpu->hm.s.vmx.cMsrs > 0)
8632 {
8633 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8634 }
8635
8636 /*
8637 * Load the host state bits as we may've been preempted (only happens when
8638 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8639 */
8640 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8641 * any effect to the host state needing to be saved? */
8642 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8643 {
8644 /* This ASSUMES that pfnStartVM has been set up already. */
8645 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8646 AssertRC(rc);
8647 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8648 }
8649 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8650
8651 /*
8652 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8653 */
8654 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8655 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8656 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8657
8658 /* Store status of the shared guest-host state at the time of VM-entry. */
8659#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8660 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8661 {
8662 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8663 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8664 }
8665 else
8666#endif
8667 {
8668 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8669 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8670 }
8671 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8672
8673 /*
8674 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8675 */
8676 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8677 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8678
8679 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8680 RTCPUID idCurrentCpu = pCpu->idCpu;
8681 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8682 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8683 {
8684 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8685 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8686 }
8687
8688 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8689 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8690 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8691 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8692
8693 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8694
8695 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8696 to start executing. */
8697
8698 /*
8699 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8700 */
8701 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8702 {
8703 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8704 {
8705 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8706 AssertRC(rc2);
8707 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8708 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8709 true /* fUpdateHostMsr */);
8710 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8711 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8712 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8713 }
8714 else
8715 {
8716 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8717 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8718 }
8719 }
8720
8721#ifdef VBOX_STRICT
8722 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8723 hmR0VmxCheckHostEferMsr(pVCpu);
8724 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8725#endif
8726#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8727 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8728 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8729 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8730#endif
8731}
8732
8733
8734/**
8735 * Performs some essential restoration of state after running guest code in
8736 * VT-x.
8737 *
8738 * @param pVM Pointer to the VM.
8739 * @param pVCpu Pointer to the VMCPU.
8740 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8741 * out-of-sync. Make sure to update the required fields
8742 * before using them.
8743 * @param pVmxTransient Pointer to the VMX transient structure.
8744 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8745 *
8746 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8747 *
8748 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8749 * unconditionally when it is safe to do so.
8750 */
8751static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8752{
8753 NOREF(pVM);
8754
8755 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8756
8757 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8758 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8759 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8760 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8761 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8762 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8763
8764 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8765 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8766
8767 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8768 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8769 Assert(!(ASMGetFlags() & X86_EFL_IF));
8770 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8771
8772#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8773 if (CPUMIsGuestFPUStateActive(pVCpu))
8774 {
8775 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8776 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8777 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8778 }
8779#endif
8780
8781#if HC_ARCH_BITS == 64
8782 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8783#endif
8784 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8785#ifdef VBOX_STRICT
8786 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8787#endif
8788 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8789 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8790
8791 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8792 uint32_t uExitReason;
8793 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8794 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8795 AssertRC(rc);
8796 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8797 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8798
8799 /* Update the VM-exit history array. */
8800 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8801
8802 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8803 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8804 {
8805 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8806 pVmxTransient->fVMEntryFailed));
8807 return;
8808 }
8809
8810 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8811 {
8812 /** @todo We can optimize this by only syncing with our force-flags when
8813 * really needed and keeping the VMCS state as it is for most
8814 * VM-exits. */
8815 /* Update the guest interruptibility-state from the VMCS. */
8816 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8817
8818#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8819 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8820 AssertRC(rc);
8821#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8822 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8823 AssertRC(rc);
8824#endif
8825
8826 /*
8827 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8828 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8829 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8830 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8831 */
8832 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8833 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8834 {
8835 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8836 AssertRC(rc);
8837 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8838 }
8839 }
8840}
8841
8842
8843/**
8844 * Runs the guest code using VT-x the normal way.
8845 *
8846 * @returns VBox status code.
8847 * @param pVM Pointer to the VM.
8848 * @param pVCpu Pointer to the VMCPU.
8849 * @param pCtx Pointer to the guest-CPU context.
8850 *
8851 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8852 */
8853static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8854{
8855 VMXTRANSIENT VmxTransient;
8856 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8857 int rc = VERR_INTERNAL_ERROR_5;
8858 uint32_t cLoops = 0;
8859
8860 for (;; cLoops++)
8861 {
8862 Assert(!HMR0SuspendPending());
8863 HMVMX_ASSERT_CPU_SAFE();
8864
8865 /* Preparatory work for running guest code, this may force us to return
8866 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8867 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8868 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8869 if (rc != VINF_SUCCESS)
8870 break;
8871
8872 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8873 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8874 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8875
8876 /* Restore any residual host-state and save any bits shared between host
8877 and guest into the guest-CPU state. Re-enables interrupts! */
8878 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8879
8880 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8881 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8882 {
8883 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8884 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8885 return rc;
8886 }
8887
8888 /* Profile the VM-exit. */
8889 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8890 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8891 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8892 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8893 HMVMX_START_EXIT_DISPATCH_PROF();
8894
8895 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8896 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8897 {
8898 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8899 hmR0VmxSaveGuestState(pVCpu, pCtx);
8900 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8901 }
8902
8903 /* Handle the VM-exit. */
8904#ifdef HMVMX_USE_FUNCTION_TABLE
8905 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8906#else
8907 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8908#endif
8909 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8910 if (rc != VINF_SUCCESS)
8911 break;
8912 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8913 {
8914 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8915 rc = VINF_EM_RAW_INTERRUPT;
8916 break;
8917 }
8918 }
8919
8920 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8921 return rc;
8922}
8923
8924
8925/**
8926 * Single steps guest code using VT-x.
8927 *
8928 * @returns VBox status code.
8929 * @param pVM Pointer to the VM.
8930 * @param pVCpu Pointer to the VMCPU.
8931 * @param pCtx Pointer to the guest-CPU context.
8932 *
8933 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8934 */
8935static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8936{
8937 VMXTRANSIENT VmxTransient;
8938 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8939 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8940 uint32_t cLoops = 0;
8941 uint16_t uCsStart = pCtx->cs.Sel;
8942 uint64_t uRipStart = pCtx->rip;
8943
8944 for (;; cLoops++)
8945 {
8946 Assert(!HMR0SuspendPending());
8947 HMVMX_ASSERT_CPU_SAFE();
8948
8949 /* Preparatory work for running guest code, this may force us to return
8950 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8951 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8952 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8953 if (rcStrict != VINF_SUCCESS)
8954 break;
8955
8956 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8957 rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8958 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8959
8960 /* Restore any residual host-state and save any bits shared between host
8961 and guest into the guest-CPU state. Re-enables interrupts! */
8962 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8963
8964 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8965 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
8966 {
8967 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8968 hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
8969 return VBOXSTRICTRC_TODO(rcStrict);
8970 }
8971
8972 /* Profile the VM-exit. */
8973 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8974 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8975 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8976 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8977 HMVMX_START_EXIT_DISPATCH_PROF();
8978
8979 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8980 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8981 {
8982 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8983 hmR0VmxSaveGuestState(pVCpu, pCtx);
8984 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8985 }
8986
8987 /* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8988 rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8989 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8990 if (rcStrict != VINF_SUCCESS)
8991 break;
8992 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8993 {
8994 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8995 rcStrict = VINF_EM_RAW_INTERRUPT;
8996 break;
8997 }
8998
8999 /*
9000 * Did the RIP change, if so, consider it a single step.
9001 * Otherwise, make sure one of the TFs gets set.
9002 */
9003 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
9004 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
9005 AssertRCReturn(rc2, rc2);
9006 if ( pCtx->rip != uRipStart
9007 || pCtx->cs.Sel != uCsStart)
9008 {
9009 rcStrict = VINF_EM_DBG_STEPPED;
9010 break;
9011 }
9012 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
9013 }
9014
9015 /*
9016 * Clear the X86_EFL_TF if necessary.
9017 */
9018 if (pVCpu->hm.s.fClearTrapFlag)
9019 {
9020 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
9021 AssertRCReturn(rc2, rc2);
9022 pVCpu->hm.s.fClearTrapFlag = false;
9023 pCtx->eflags.Bits.u1TF = 0;
9024 }
9025 /** @todo there seems to be issues with the resume flag when the monitor trap
9026 * flag is pending without being used. Seen early in bios init when
9027 * accessing APIC page in protected mode. */
9028
9029 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9030 return VBOXSTRICTRC_TODO(rcStrict);
9031}
9032
9033
9034/**
9035 * Runs the guest code using VT-x.
9036 *
9037 * @returns VBox status code.
9038 * @param pVM Pointer to the VM.
9039 * @param pVCpu Pointer to the VMCPU.
9040 * @param pCtx Pointer to the guest-CPU context.
9041 */
9042VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9043{
9044 Assert(VMMRZCallRing3IsEnabled(pVCpu));
9045 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
9046 HMVMX_ASSERT_PREEMPT_SAFE();
9047
9048 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
9049
9050 int rc;
9051 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
9052 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
9053 else
9054 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
9055
9056 if (rc == VERR_EM_INTERPRETER)
9057 rc = VINF_EM_RAW_EMULATE_INSTR;
9058 else if (rc == VINF_EM_RESET)
9059 rc = VINF_EM_TRIPLE_FAULT;
9060
9061 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
9062 if (RT_FAILURE(rc2))
9063 {
9064 pVCpu->hm.s.u32HMError = rc;
9065 rc = rc2;
9066 }
9067 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
9068 return rc;
9069}
9070
9071
9072#ifndef HMVMX_USE_FUNCTION_TABLE
9073DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9074{
9075#ifdef DEBUG_ramshankar
9076# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9077# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9078#endif
9079 int rc;
9080 switch (rcReason)
9081 {
9082 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9083 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9084 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9085 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9086 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9087 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9088 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9089 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9090 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9091 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9092 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9093 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9094 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9095 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9096 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9097 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9098 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9099 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9100 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9101 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9102 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9103 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9104 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9105 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9106 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9107 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9108 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9109 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9110 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9111 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9112 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9113 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9114 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9115 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9116
9117 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9118 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9119 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9120 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9121 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9122 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9123 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9124 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9125 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9126
9127 case VMX_EXIT_VMCLEAR:
9128 case VMX_EXIT_VMLAUNCH:
9129 case VMX_EXIT_VMPTRLD:
9130 case VMX_EXIT_VMPTRST:
9131 case VMX_EXIT_VMREAD:
9132 case VMX_EXIT_VMRESUME:
9133 case VMX_EXIT_VMWRITE:
9134 case VMX_EXIT_VMXOFF:
9135 case VMX_EXIT_VMXON:
9136 case VMX_EXIT_INVEPT:
9137 case VMX_EXIT_INVVPID:
9138 case VMX_EXIT_VMFUNC:
9139 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9140 break;
9141 default:
9142 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9143 break;
9144 }
9145 return rc;
9146}
9147#endif /* !HMVMX_USE_FUNCTION_TABLE */
9148
9149
9150/**
9151 * Single-stepping VM-exit filtering.
9152 *
9153 * This is preprocessing the exits and deciding whether we've gotten far enough
9154 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9155 * performed.
9156 *
9157 * @returns Strict VBox status code.
9158 * @param pVCpu The virtual CPU of the calling EMT.
9159 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9160 * out-of-sync. Make sure to update the required
9161 * fields before using them.
9162 * @param pVmxTransient Pointer to the VMX-transient structure.
9163 * @param uExitReason The VM-exit reason.
9164 */
9165DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9166 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9167{
9168 switch (uExitReason)
9169 {
9170 case VMX_EXIT_XCPT_OR_NMI:
9171 {
9172 /* Check for host NMI. */
9173 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9174 AssertRCReturn(rc2, rc2);
9175 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9176 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9177 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9178 /* fall thru */
9179 }
9180
9181 case VMX_EXIT_EPT_MISCONFIG:
9182 case VMX_EXIT_TRIPLE_FAULT:
9183 case VMX_EXIT_APIC_ACCESS:
9184 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9185 case VMX_EXIT_TASK_SWITCH:
9186
9187 /* Instruction specific VM-exits: */
9188 case VMX_EXIT_IO_INSTR:
9189 case VMX_EXIT_CPUID:
9190 case VMX_EXIT_RDTSC:
9191 case VMX_EXIT_RDTSCP:
9192 case VMX_EXIT_MOV_CRX:
9193 case VMX_EXIT_MWAIT:
9194 case VMX_EXIT_MONITOR:
9195 case VMX_EXIT_RDMSR:
9196 case VMX_EXIT_WRMSR:
9197 case VMX_EXIT_MOV_DRX:
9198 case VMX_EXIT_HLT:
9199 case VMX_EXIT_INVD:
9200 case VMX_EXIT_INVLPG:
9201 case VMX_EXIT_RSM:
9202 case VMX_EXIT_PAUSE:
9203 case VMX_EXIT_XDTR_ACCESS:
9204 case VMX_EXIT_TR_ACCESS:
9205 case VMX_EXIT_WBINVD:
9206 case VMX_EXIT_XSETBV:
9207 case VMX_EXIT_RDRAND:
9208 case VMX_EXIT_INVPCID:
9209 case VMX_EXIT_GETSEC:
9210 case VMX_EXIT_RDPMC:
9211 case VMX_EXIT_VMCALL:
9212 case VMX_EXIT_VMCLEAR:
9213 case VMX_EXIT_VMLAUNCH:
9214 case VMX_EXIT_VMPTRLD:
9215 case VMX_EXIT_VMPTRST:
9216 case VMX_EXIT_VMREAD:
9217 case VMX_EXIT_VMRESUME:
9218 case VMX_EXIT_VMWRITE:
9219 case VMX_EXIT_VMXOFF:
9220 case VMX_EXIT_VMXON:
9221 case VMX_EXIT_INVEPT:
9222 case VMX_EXIT_INVVPID:
9223 case VMX_EXIT_VMFUNC:
9224 {
9225 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9226 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9227 AssertRCReturn(rc2, rc2);
9228 if ( pMixedCtx->rip != uRipStart
9229 || pMixedCtx->cs.Sel != uCsStart)
9230 return VINF_EM_DBG_STEPPED;
9231 break;
9232 }
9233 }
9234
9235 /*
9236 * Normal processing.
9237 */
9238#ifdef HMVMX_USE_FUNCTION_TABLE
9239 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9240#else
9241 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9242#endif
9243}
9244
9245
9246#ifdef DEBUG
9247/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9248# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9249 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9250
9251# define HMVMX_ASSERT_PREEMPT_CPUID() \
9252 do \
9253 { \
9254 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9255 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9256 } while (0)
9257
9258# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9259 do { \
9260 AssertPtr(pVCpu); \
9261 AssertPtr(pMixedCtx); \
9262 AssertPtr(pVmxTransient); \
9263 Assert(pVmxTransient->fVMEntryFailed == false); \
9264 Assert(ASMIntAreEnabled()); \
9265 HMVMX_ASSERT_PREEMPT_SAFE(); \
9266 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9267 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9268 HMVMX_ASSERT_PREEMPT_SAFE(); \
9269 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9270 HMVMX_ASSERT_PREEMPT_CPUID(); \
9271 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9272 } while (0)
9273
9274# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9275 do { \
9276 Log4Func(("\n")); \
9277 } while (0)
9278#else /* Release builds */
9279# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9280 do { \
9281 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9282 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9283 } while (0)
9284# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9285#endif
9286
9287
9288/**
9289 * Advances the guest RIP after reading it from the VMCS.
9290 *
9291 * @returns VBox status code.
9292 * @param pVCpu Pointer to the VMCPU.
9293 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9294 * out-of-sync. Make sure to update the required fields
9295 * before using them.
9296 * @param pVmxTransient Pointer to the VMX transient structure.
9297 *
9298 * @remarks No-long-jump zone!!!
9299 */
9300DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9301{
9302 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9303 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9304 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9305 AssertRCReturn(rc, rc);
9306
9307 pMixedCtx->rip += pVmxTransient->cbInstr;
9308 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9309
9310 /*
9311 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9312 * pending debug exception field as it takes care of priority of events.
9313 *
9314 * See Intel spec. 32.2.1 "Debug Exceptions".
9315 */
9316 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9317
9318 return rc;
9319}
9320
9321
9322/**
9323 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9324 * and update error record fields accordingly.
9325 *
9326 * @return VMX_IGS_* return codes.
9327 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9328 * wrong with the guest state.
9329 *
9330 * @param pVM Pointer to the VM.
9331 * @param pVCpu Pointer to the VMCPU.
9332 * @param pCtx Pointer to the guest-CPU state.
9333 *
9334 * @remarks This function assumes our cache of the VMCS controls
9335 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9336 */
9337static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9338{
9339#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9340#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9341 uError = (err); \
9342 break; \
9343 } else do { } while (0)
9344
9345 int rc;
9346 uint32_t uError = VMX_IGS_ERROR;
9347 uint32_t u32Val;
9348 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9349
9350 do
9351 {
9352 /*
9353 * CR0.
9354 */
9355 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9356 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9357 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9358 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9359 if (fUnrestrictedGuest)
9360 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9361
9362 uint32_t u32GuestCR0;
9363 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9364 AssertRCBreak(rc);
9365 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9366 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9367 if ( !fUnrestrictedGuest
9368 && (u32GuestCR0 & X86_CR0_PG)
9369 && !(u32GuestCR0 & X86_CR0_PE))
9370 {
9371 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9372 }
9373
9374 /*
9375 * CR4.
9376 */
9377 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9378 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9379
9380 uint32_t u32GuestCR4;
9381 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9382 AssertRCBreak(rc);
9383 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9384 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9385
9386 /*
9387 * IA32_DEBUGCTL MSR.
9388 */
9389 uint64_t u64Val;
9390 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9391 AssertRCBreak(rc);
9392 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9393 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9394 {
9395 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9396 }
9397 uint64_t u64DebugCtlMsr = u64Val;
9398
9399#ifdef VBOX_STRICT
9400 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9401 AssertRCBreak(rc);
9402 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9403#endif
9404 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9405
9406 /*
9407 * RIP and RFLAGS.
9408 */
9409 uint32_t u32Eflags;
9410#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9411 if (HMVMX_IS_64BIT_HOST_MODE())
9412 {
9413 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9414 AssertRCBreak(rc);
9415 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9416 if ( !fLongModeGuest
9417 || !pCtx->cs.Attr.n.u1Long)
9418 {
9419 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9420 }
9421 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9422 * must be identical if the "IA-32e mode guest" VM-entry
9423 * control is 1 and CS.L is 1. No check applies if the
9424 * CPU supports 64 linear-address bits. */
9425
9426 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9427 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9428 AssertRCBreak(rc);
9429 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9430 VMX_IGS_RFLAGS_RESERVED);
9431 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9432 u32Eflags = u64Val;
9433 }
9434 else
9435#endif
9436 {
9437 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9438 AssertRCBreak(rc);
9439 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9440 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9441 }
9442
9443 if ( fLongModeGuest
9444 || ( fUnrestrictedGuest
9445 && !(u32GuestCR0 & X86_CR0_PE)))
9446 {
9447 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9448 }
9449
9450 uint32_t u32EntryInfo;
9451 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9452 AssertRCBreak(rc);
9453 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9454 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9455 {
9456 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9457 }
9458
9459 /*
9460 * 64-bit checks.
9461 */
9462#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9463 if (HMVMX_IS_64BIT_HOST_MODE())
9464 {
9465 if ( fLongModeGuest
9466 && !fUnrestrictedGuest)
9467 {
9468 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9469 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9470 }
9471
9472 if ( !fLongModeGuest
9473 && (u32GuestCR4 & X86_CR4_PCIDE))
9474 {
9475 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9476 }
9477
9478 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9479 * 51:32 beyond the processor's physical-address width are 0. */
9480
9481 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9482 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9483 {
9484 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9485 }
9486
9487 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9488 AssertRCBreak(rc);
9489 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9490
9491 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9492 AssertRCBreak(rc);
9493 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9494 }
9495#endif
9496
9497 /*
9498 * PERF_GLOBAL MSR.
9499 */
9500 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9501 {
9502 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9503 AssertRCBreak(rc);
9504 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9505 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9506 }
9507
9508 /*
9509 * PAT MSR.
9510 */
9511 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9512 {
9513 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9514 AssertRCBreak(rc);
9515 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9516 for (unsigned i = 0; i < 8; i++)
9517 {
9518 uint8_t u8Val = (u64Val & 0xff);
9519 if ( u8Val != 0 /* UC */
9520 && u8Val != 1 /* WC */
9521 && u8Val != 4 /* WT */
9522 && u8Val != 5 /* WP */
9523 && u8Val != 6 /* WB */
9524 && u8Val != 7 /* UC- */)
9525 {
9526 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9527 }
9528 u64Val >>= 8;
9529 }
9530 }
9531
9532 /*
9533 * EFER MSR.
9534 */
9535 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9536 {
9537 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9538 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9539 AssertRCBreak(rc);
9540 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9541 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9542 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9543 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9544 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9545 || !(u32GuestCR0 & X86_CR0_PG)
9546 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9547 VMX_IGS_EFER_LMA_LME_MISMATCH);
9548 }
9549
9550 /*
9551 * Segment registers.
9552 */
9553 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9554 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9555 if (!(u32Eflags & X86_EFL_VM))
9556 {
9557 /* CS */
9558 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9559 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9560 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9561 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9562 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9563 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9564 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9565 /* CS cannot be loaded with NULL in protected mode. */
9566 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9567 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9568 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9569 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9570 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9571 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9572 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9573 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9574 else
9575 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9576
9577 /* SS */
9578 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9579 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9580 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9581 if ( !(pCtx->cr0 & X86_CR0_PE)
9582 || pCtx->cs.Attr.n.u4Type == 3)
9583 {
9584 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9585 }
9586 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9587 {
9588 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9589 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9590 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9591 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9592 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9593 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9594 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9595 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9596 }
9597
9598 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9599 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9600 {
9601 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9602 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9603 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9604 || pCtx->ds.Attr.n.u4Type > 11
9605 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9606 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9607 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9608 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9609 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9610 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9611 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9612 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9613 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9614 }
9615 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9616 {
9617 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9618 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9619 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9620 || pCtx->es.Attr.n.u4Type > 11
9621 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9622 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9623 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9624 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9625 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9626 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9627 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9628 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9629 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9630 }
9631 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9632 {
9633 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9634 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9635 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9636 || pCtx->fs.Attr.n.u4Type > 11
9637 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9638 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9639 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9640 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9641 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9642 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9643 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9644 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9645 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9646 }
9647 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9648 {
9649 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9650 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9651 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9652 || pCtx->gs.Attr.n.u4Type > 11
9653 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9654 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9655 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9656 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9657 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9658 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9659 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9660 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9661 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9662 }
9663 /* 64-bit capable CPUs. */
9664#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9665 if (HMVMX_IS_64BIT_HOST_MODE())
9666 {
9667 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9668 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9669 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9670 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9671 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9672 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9673 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9674 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9675 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9676 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9677 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9678 }
9679#endif
9680 }
9681 else
9682 {
9683 /* V86 mode checks. */
9684 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9685 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9686 {
9687 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9688 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9689 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9690 }
9691 else
9692 {
9693 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9694 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9695 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9696 }
9697
9698 /* CS */
9699 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9700 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9701 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9702 /* SS */
9703 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9704 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9705 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9706 /* DS */
9707 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9708 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9709 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9710 /* ES */
9711 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9712 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9713 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9714 /* FS */
9715 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9716 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9717 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9718 /* GS */
9719 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9720 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9721 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9722 /* 64-bit capable CPUs. */
9723#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9724 if (HMVMX_IS_64BIT_HOST_MODE())
9725 {
9726 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9727 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9728 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9729 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9730 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9731 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9732 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9733 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9734 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9735 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9736 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9737 }
9738#endif
9739 }
9740
9741 /*
9742 * TR.
9743 */
9744 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9745 /* 64-bit capable CPUs. */
9746#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9747 if (HMVMX_IS_64BIT_HOST_MODE())
9748 {
9749 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9750 }
9751#endif
9752 if (fLongModeGuest)
9753 {
9754 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9755 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9756 }
9757 else
9758 {
9759 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9760 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9761 VMX_IGS_TR_ATTR_TYPE_INVALID);
9762 }
9763 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9764 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9765 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9766 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9767 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9768 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9769 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9770 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9771
9772 /*
9773 * GDTR and IDTR.
9774 */
9775#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9776 if (HMVMX_IS_64BIT_HOST_MODE())
9777 {
9778 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9779 AssertRCBreak(rc);
9780 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9781
9782 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9783 AssertRCBreak(rc);
9784 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9785 }
9786#endif
9787
9788 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9789 AssertRCBreak(rc);
9790 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9791
9792 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9793 AssertRCBreak(rc);
9794 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9795
9796 /*
9797 * Guest Non-Register State.
9798 */
9799 /* Activity State. */
9800 uint32_t u32ActivityState;
9801 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9802 AssertRCBreak(rc);
9803 HMVMX_CHECK_BREAK( !u32ActivityState
9804 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9805 VMX_IGS_ACTIVITY_STATE_INVALID);
9806 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9807 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9808 uint32_t u32IntrState;
9809 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9810 AssertRCBreak(rc);
9811 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9812 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9813 {
9814 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9815 }
9816
9817 /** @todo Activity state and injecting interrupts. Left as a todo since we
9818 * currently don't use activity states but ACTIVE. */
9819
9820 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9821 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9822
9823 /* Guest interruptibility-state. */
9824 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9825 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9826 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9827 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9828 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9829 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9830 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9831 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9832 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9833 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9834 {
9835 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9836 {
9837 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9838 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9839 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9840 }
9841 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9842 {
9843 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9844 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9845 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9846 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9847 }
9848 }
9849 /** @todo Assumes the processor is not in SMM. */
9850 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9851 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9852 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9853 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9854 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9855 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9856 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9857 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9858 {
9859 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9860 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9861 }
9862
9863 /* Pending debug exceptions. */
9864 if (HMVMX_IS_64BIT_HOST_MODE())
9865 {
9866 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9867 AssertRCBreak(rc);
9868 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9869 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9870 u32Val = u64Val; /* For pending debug exceptions checks below. */
9871 }
9872 else
9873 {
9874 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9875 AssertRCBreak(rc);
9876 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9877 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9878 }
9879
9880 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9881 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9882 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9883 {
9884 if ( (u32Eflags & X86_EFL_TF)
9885 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9886 {
9887 /* Bit 14 is PendingDebug.BS. */
9888 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9889 }
9890 if ( !(u32Eflags & X86_EFL_TF)
9891 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9892 {
9893 /* Bit 14 is PendingDebug.BS. */
9894 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9895 }
9896 }
9897
9898 /* VMCS link pointer. */
9899 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9900 AssertRCBreak(rc);
9901 if (u64Val != UINT64_C(0xffffffffffffffff))
9902 {
9903 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9904 /** @todo Bits beyond the processor's physical-address width MBZ. */
9905 /** @todo 32-bit located in memory referenced by value of this field (as a
9906 * physical address) must contain the processor's VMCS revision ID. */
9907 /** @todo SMM checks. */
9908 }
9909
9910 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9911 * not using Nested Paging? */
9912 if ( pVM->hm.s.fNestedPaging
9913 && !fLongModeGuest
9914 && CPUMIsGuestInPAEModeEx(pCtx))
9915 {
9916 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9917 AssertRCBreak(rc);
9918 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9919
9920 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9921 AssertRCBreak(rc);
9922 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9923
9924 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9925 AssertRCBreak(rc);
9926 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9927
9928 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9929 AssertRCBreak(rc);
9930 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9931 }
9932
9933 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9934 if (uError == VMX_IGS_ERROR)
9935 uError = VMX_IGS_REASON_NOT_FOUND;
9936 } while (0);
9937
9938 pVCpu->hm.s.u32HMError = uError;
9939 return uError;
9940
9941#undef HMVMX_ERROR_BREAK
9942#undef HMVMX_CHECK_BREAK
9943}
9944
9945/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9946/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9947/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9948
9949/** @name VM-exit handlers.
9950 * @{
9951 */
9952
9953/**
9954 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9955 */
9956HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9957{
9958 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9959 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9960 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9961 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9962 return VINF_SUCCESS;
9963 return VINF_EM_RAW_INTERRUPT;
9964}
9965
9966
9967/**
9968 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9969 */
9970HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9971{
9972 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9973 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9974
9975 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9976 AssertRCReturn(rc, rc);
9977
9978 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9979 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9980 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9981 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9982
9983 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9984 {
9985 /*
9986 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9987 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9988 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9989 *
9990 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9991 */
9992 VMXDispatchHostNmi();
9993 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9994 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9995 return VINF_SUCCESS;
9996 }
9997
9998 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9999 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10000 if (RT_UNLIKELY(rc != VINF_SUCCESS))
10001 {
10002 if (rc == VINF_HM_DOUBLE_FAULT)
10003 rc = VINF_SUCCESS;
10004 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10005 return rc;
10006 }
10007
10008 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10009 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10010 switch (uIntType)
10011 {
10012 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10013 Assert(uVector == X86_XCPT_DB);
10014 /* no break */
10015 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10016 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10017 /* no break */
10018 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10019 {
10020 switch (uVector)
10021 {
10022 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
10023 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
10024 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
10025 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
10026 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
10027 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
10028#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10029 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10030 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10031 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10032 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10033 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10034 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10035 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10036 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10037 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10038 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10039 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10040 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
10041#endif
10042 default:
10043 {
10044 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10045 AssertRCReturn(rc, rc);
10046
10047 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10048 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10049 {
10050 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10051 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10052 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10053
10054 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10055 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10056 AssertRCReturn(rc, rc);
10057 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10058 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10059 0 /* GCPtrFaultAddress */);
10060 AssertRCReturn(rc, rc);
10061 }
10062 else
10063 {
10064 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10065 pVCpu->hm.s.u32HMError = uVector;
10066 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10067 }
10068 break;
10069 }
10070 }
10071 break;
10072 }
10073
10074 default:
10075 {
10076 pVCpu->hm.s.u32HMError = uExitIntInfo;
10077 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10078 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10079 break;
10080 }
10081 }
10082 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10083 return rc;
10084}
10085
10086
10087/**
10088 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10089 */
10090HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10091{
10092 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10093
10094 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
10095 hmR0VmxClearIntWindowExitVmcs(pVCpu);
10096
10097 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10098 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
10099 return VINF_SUCCESS;
10100}
10101
10102
10103/**
10104 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
10105 */
10106HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10107{
10108 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10109 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
10110 {
10111 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
10112 HMVMX_RETURN_UNEXPECTED_EXIT();
10113 }
10114
10115 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
10116
10117 /*
10118 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
10119 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
10120 */
10121 uint32_t uIntrState = 0;
10122 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10123 AssertRCReturn(rc, rc);
10124
10125 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
10126 if ( fBlockSti
10127 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
10128 {
10129 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10130 }
10131
10132 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
10133 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
10134
10135 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10136 return VINF_SUCCESS;
10137}
10138
10139
10140/**
10141 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10142 */
10143HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10144{
10145 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10146 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10147 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10148}
10149
10150
10151/**
10152 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10153 */
10154HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10155{
10156 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10157 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10158 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10159}
10160
10161
10162/**
10163 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10164 */
10165HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10166{
10167 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10168 PVM pVM = pVCpu->CTX_SUFF(pVM);
10169 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10170 if (RT_LIKELY(rc == VINF_SUCCESS))
10171 {
10172 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10173 Assert(pVmxTransient->cbInstr == 2);
10174 }
10175 else
10176 {
10177 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10178 rc = VERR_EM_INTERPRETER;
10179 }
10180 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10181 return rc;
10182}
10183
10184
10185/**
10186 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10187 */
10188HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10189{
10190 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10191 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10192 AssertRCReturn(rc, rc);
10193
10194 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10195 return VINF_EM_RAW_EMULATE_INSTR;
10196
10197 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10198 HMVMX_RETURN_UNEXPECTED_EXIT();
10199}
10200
10201
10202/**
10203 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10204 */
10205HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10206{
10207 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10208 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10209 AssertRCReturn(rc, rc);
10210
10211 PVM pVM = pVCpu->CTX_SUFF(pVM);
10212 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10213 if (RT_LIKELY(rc == VINF_SUCCESS))
10214 {
10215 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10216 Assert(pVmxTransient->cbInstr == 2);
10217 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10218 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10219 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10220 }
10221 else
10222 rc = VERR_EM_INTERPRETER;
10223 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10224 return rc;
10225}
10226
10227
10228/**
10229 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10230 */
10231HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10232{
10233 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10234 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10235 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10236 AssertRCReturn(rc, rc);
10237
10238 PVM pVM = pVCpu->CTX_SUFF(pVM);
10239 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10240 if (RT_LIKELY(rc == VINF_SUCCESS))
10241 {
10242 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10243 Assert(pVmxTransient->cbInstr == 3);
10244 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10245 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10246 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10247 }
10248 else
10249 {
10250 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10251 rc = VERR_EM_INTERPRETER;
10252 }
10253 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10254 return rc;
10255}
10256
10257
10258/**
10259 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10260 */
10261HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10262{
10263 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10264 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10265 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10266 AssertRCReturn(rc, rc);
10267
10268 PVM pVM = pVCpu->CTX_SUFF(pVM);
10269 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10270 if (RT_LIKELY(rc == VINF_SUCCESS))
10271 {
10272 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10273 Assert(pVmxTransient->cbInstr == 2);
10274 }
10275 else
10276 {
10277 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10278 rc = VERR_EM_INTERPRETER;
10279 }
10280 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10281 return rc;
10282}
10283
10284
10285/**
10286 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10287 */
10288HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10289{
10290 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10291 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10292
10293 if (pVCpu->hm.s.fHypercallsEnabled)
10294 {
10295#if 0
10296 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10297 AssertRCReturn(rc, rc);
10298#else
10299 /* Aggressive state sync. for now. */
10300 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10301 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
10302#endif
10303 AssertRCReturn(rc, rc);
10304
10305 rc = GIMHypercall(pVCpu, pMixedCtx);
10306 if (RT_SUCCESS(rc))
10307 {
10308 /* If the hypercall changes anything other than guest general-purpose registers,
10309 we would need to reload the guest changed bits here before VM-reentry. */
10310 hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10311 return VINF_SUCCESS;
10312 }
10313 }
10314
10315 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10316 return VINF_SUCCESS;
10317}
10318
10319
10320/**
10321 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10322 */
10323HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10324{
10325 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10326 PVM pVM = pVCpu->CTX_SUFF(pVM);
10327 Assert(!pVM->hm.s.fNestedPaging);
10328
10329 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10330 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10331 AssertRCReturn(rc, rc);
10332
10333 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10334 rc = VBOXSTRICTRC_VAL(rc2);
10335 if (RT_LIKELY(rc == VINF_SUCCESS))
10336 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10337 else
10338 {
10339 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10340 pVmxTransient->uExitQualification, rc));
10341 }
10342 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10343 return rc;
10344}
10345
10346
10347/**
10348 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10349 */
10350HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10351{
10352 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10353 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10354 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10355 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10356 AssertRCReturn(rc, rc);
10357
10358 PVM pVM = pVCpu->CTX_SUFF(pVM);
10359 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10360 if (RT_LIKELY(rc == VINF_SUCCESS))
10361 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10362 else
10363 {
10364 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10365 rc = VERR_EM_INTERPRETER;
10366 }
10367 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10368 return rc;
10369}
10370
10371
10372/**
10373 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10374 */
10375HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10376{
10377 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10378 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10379 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10380 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10381 AssertRCReturn(rc, rc);
10382
10383 PVM pVM = pVCpu->CTX_SUFF(pVM);
10384 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10385 rc = VBOXSTRICTRC_VAL(rc2);
10386 if (RT_LIKELY( rc == VINF_SUCCESS
10387 || rc == VINF_EM_HALT))
10388 {
10389 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10390 AssertRCReturn(rc3, rc3);
10391
10392 if ( rc == VINF_EM_HALT
10393 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10394 {
10395 rc = VINF_SUCCESS;
10396 }
10397 }
10398 else
10399 {
10400 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10401 rc = VERR_EM_INTERPRETER;
10402 }
10403 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10404 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10405 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10406 return rc;
10407}
10408
10409
10410/**
10411 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10412 */
10413HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10414{
10415 /*
10416 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10417 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10418 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10419 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10420 */
10421 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10422 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10423 HMVMX_RETURN_UNEXPECTED_EXIT();
10424}
10425
10426
10427/**
10428 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10429 */
10430HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10431{
10432 /*
10433 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10434 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10435 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10436 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10437 */
10438 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10439 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10440 HMVMX_RETURN_UNEXPECTED_EXIT();
10441}
10442
10443
10444/**
10445 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10446 */
10447HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10448{
10449 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10450 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10451 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10452 HMVMX_RETURN_UNEXPECTED_EXIT();
10453}
10454
10455
10456/**
10457 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10458 */
10459HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10460{
10461 /*
10462 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10463 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10464 * See Intel spec. 25.3 "Other Causes of VM-exits".
10465 */
10466 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10467 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10468 HMVMX_RETURN_UNEXPECTED_EXIT();
10469}
10470
10471
10472/**
10473 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10474 * VM-exit.
10475 */
10476HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10477{
10478 /*
10479 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10480 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10481 *
10482 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10483 * See Intel spec. "23.8 Restrictions on VMX operation".
10484 */
10485 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10486 return VINF_SUCCESS;
10487}
10488
10489
10490/**
10491 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10492 * VM-exit.
10493 */
10494HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10495{
10496 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10497 return VINF_EM_RESET;
10498}
10499
10500
10501/**
10502 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10503 */
10504HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10505{
10506 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10507 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10508 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10509 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10510 AssertRCReturn(rc, rc);
10511
10512 pMixedCtx->rip++;
10513 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10514 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10515 rc = VINF_SUCCESS;
10516 else
10517 rc = VINF_EM_HALT;
10518
10519 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10520 if (rc != VINF_SUCCESS)
10521 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
10522 return rc;
10523}
10524
10525
10526/**
10527 * VM-exit handler for instructions that result in a #UD exception delivered to
10528 * the guest.
10529 */
10530HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10531{
10532 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10533 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10534 return VINF_SUCCESS;
10535}
10536
10537
10538/**
10539 * VM-exit handler for expiry of the VMX preemption timer.
10540 */
10541HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10542{
10543 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10544
10545 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10546 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10547
10548 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10549 PVM pVM = pVCpu->CTX_SUFF(pVM);
10550 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10551 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10552 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10553}
10554
10555
10556/**
10557 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10558 */
10559HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10560{
10561 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10562
10563 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10564 /** @todo check if XSETBV is supported by the recompiler. */
10565 return VERR_EM_INTERPRETER;
10566}
10567
10568
10569/**
10570 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10571 */
10572HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10573{
10574 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10575
10576 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10577 /** @todo implement EMInterpretInvpcid() */
10578 return VERR_EM_INTERPRETER;
10579}
10580
10581
10582/**
10583 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10584 * Error VM-exit.
10585 */
10586HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10587{
10588 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10589 AssertRCReturn(rc, rc);
10590
10591 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10592 AssertRCReturn(rc, rc);
10593
10594 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10595 NOREF(uInvalidReason);
10596
10597#ifdef VBOX_STRICT
10598 uint32_t uIntrState;
10599 HMVMXHCUINTREG uHCReg;
10600 uint64_t u64Val;
10601 uint32_t u32Val;
10602
10603 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10604 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10605 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10606 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10607 AssertRCReturn(rc, rc);
10608
10609 Log4(("uInvalidReason %u\n", uInvalidReason));
10610 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10611 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10612 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10613 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10614
10615 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10616 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10617 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10618 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10619 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10620 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10621 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10622 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10623 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10624 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10625 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10626 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10627#else
10628 NOREF(pVmxTransient);
10629#endif
10630
10631 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10632 return VERR_VMX_INVALID_GUEST_STATE;
10633}
10634
10635
10636/**
10637 * VM-exit handler for VM-entry failure due to an MSR-load
10638 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10639 */
10640HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10641{
10642 NOREF(pVmxTransient);
10643 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10644 HMVMX_RETURN_UNEXPECTED_EXIT();
10645}
10646
10647
10648/**
10649 * VM-exit handler for VM-entry failure due to a machine-check event
10650 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10651 */
10652HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10653{
10654 NOREF(pVmxTransient);
10655 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10656 HMVMX_RETURN_UNEXPECTED_EXIT();
10657}
10658
10659
10660/**
10661 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10662 * theory.
10663 */
10664HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10665{
10666 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10667 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10668 return VERR_VMX_UNDEFINED_EXIT_CODE;
10669}
10670
10671
10672/**
10673 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10674 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10675 * Conditional VM-exit.
10676 */
10677HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10678{
10679 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10680
10681 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10682 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10683 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10684 return VERR_EM_INTERPRETER;
10685 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10686 HMVMX_RETURN_UNEXPECTED_EXIT();
10687}
10688
10689
10690/**
10691 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10692 */
10693HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10694{
10695 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10696
10697 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10698 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10699 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10700 return VERR_EM_INTERPRETER;
10701 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10702 HMVMX_RETURN_UNEXPECTED_EXIT();
10703}
10704
10705
10706/**
10707 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10708 */
10709HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10710{
10711 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10712
10713 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10714 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10715 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10716 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10717 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10718 {
10719 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10720 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10721 }
10722 AssertRCReturn(rc, rc);
10723 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10724
10725#ifdef VBOX_STRICT
10726 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10727 {
10728 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10729 && pMixedCtx->ecx != MSR_K6_EFER)
10730 {
10731 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10732 HMVMX_RETURN_UNEXPECTED_EXIT();
10733 }
10734# if HC_ARCH_BITS == 64
10735 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10736 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10737 {
10738 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10739 HMVMX_RETURN_UNEXPECTED_EXIT();
10740 }
10741# endif
10742 }
10743#endif
10744
10745 PVM pVM = pVCpu->CTX_SUFF(pVM);
10746 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10747 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10748 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10749 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10750 if (RT_LIKELY(rc == VINF_SUCCESS))
10751 {
10752 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10753 Assert(pVmxTransient->cbInstr == 2);
10754 }
10755 return rc;
10756}
10757
10758
10759/**
10760 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10761 */
10762HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10763{
10764 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10765 PVM pVM = pVCpu->CTX_SUFF(pVM);
10766 int rc = VINF_SUCCESS;
10767
10768 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10769 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10770 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10771 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10772 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10773 {
10774 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10775 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10776 }
10777 AssertRCReturn(rc, rc);
10778 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10779
10780 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10781 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10782 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10783
10784 if (RT_LIKELY(rc == VINF_SUCCESS))
10785 {
10786 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10787
10788 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10789 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10790 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10791 {
10792 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10793 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10794 EMInterpretWrmsr() changes it. */
10795 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10796 }
10797 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10798 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10799 else if (pMixedCtx->ecx == MSR_K6_EFER)
10800 {
10801 /*
10802 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10803 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10804 * the other bits as well, SCE and NXE. See @bugref{7368}.
10805 */
10806 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10807 }
10808
10809 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10810 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10811 {
10812 switch (pMixedCtx->ecx)
10813 {
10814 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10815 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10816 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10817 case MSR_K8_FS_BASE: /* no break */
10818 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10819 case MSR_K6_EFER: /* already handled above */ break;
10820 default:
10821 {
10822 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10823 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10824#if HC_ARCH_BITS == 64
10825 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10826 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10827#endif
10828 break;
10829 }
10830 }
10831 }
10832#ifdef VBOX_STRICT
10833 else
10834 {
10835 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10836 switch (pMixedCtx->ecx)
10837 {
10838 case MSR_IA32_SYSENTER_CS:
10839 case MSR_IA32_SYSENTER_EIP:
10840 case MSR_IA32_SYSENTER_ESP:
10841 case MSR_K8_FS_BASE:
10842 case MSR_K8_GS_BASE:
10843 {
10844 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10845 HMVMX_RETURN_UNEXPECTED_EXIT();
10846 }
10847
10848 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10849 default:
10850 {
10851 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10852 {
10853 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10854 if (pMixedCtx->ecx != MSR_K6_EFER)
10855 {
10856 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10857 pMixedCtx->ecx));
10858 HMVMX_RETURN_UNEXPECTED_EXIT();
10859 }
10860 }
10861
10862#if HC_ARCH_BITS == 64
10863 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10864 {
10865 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10866 HMVMX_RETURN_UNEXPECTED_EXIT();
10867 }
10868#endif
10869 break;
10870 }
10871 }
10872 }
10873#endif /* VBOX_STRICT */
10874 }
10875 return rc;
10876}
10877
10878
10879/**
10880 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10881 */
10882HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10883{
10884 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10885
10886 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10887 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10888 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10889 return VERR_EM_INTERPRETER;
10890 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10891 HMVMX_RETURN_UNEXPECTED_EXIT();
10892}
10893
10894
10895/**
10896 * VM-exit handler for when the TPR value is lowered below the specified
10897 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10898 */
10899HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10900{
10901 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10902 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10903
10904 /*
10905 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10906 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10907 * resume guest execution.
10908 */
10909 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10910 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10911 return VINF_SUCCESS;
10912}
10913
10914
10915/**
10916 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10917 * VM-exit.
10918 *
10919 * @retval VINF_SUCCESS when guest execution can continue.
10920 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10921 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10922 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10923 * recompiler.
10924 */
10925HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10926{
10927 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10928 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10929 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10930 AssertRCReturn(rc, rc);
10931
10932 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10933 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10934 PVM pVM = pVCpu->CTX_SUFF(pVM);
10935 switch (uAccessType)
10936 {
10937 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10938 {
10939#if 0
10940 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10941 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10942#else
10943 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10944 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10945 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10946#endif
10947 AssertRCReturn(rc, rc);
10948
10949 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10950 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10951 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10952 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10953
10954 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10955 {
10956 case 0: /* CR0 */
10957 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10958 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10959 break;
10960 case 2: /* CR2 */
10961 /* Nothing to do here, CR2 it's not part of the VMCS. */
10962 break;
10963 case 3: /* CR3 */
10964 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10965 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10966 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10967 break;
10968 case 4: /* CR4 */
10969 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10970 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10971 break;
10972 case 8: /* CR8 */
10973 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10974 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10975 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10976 break;
10977 default:
10978 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10979 break;
10980 }
10981
10982 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10983 break;
10984 }
10985
10986 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10987 {
10988 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10989 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10990 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10991 AssertRCReturn(rc, rc);
10992 Assert( !pVM->hm.s.fNestedPaging
10993 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10994 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10995
10996 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10997 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10998 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10999
11000 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11001 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
11002 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
11003 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11004 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
11005 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
11006 break;
11007 }
11008
11009 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
11010 {
11011 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11012 AssertRCReturn(rc, rc);
11013 rc = EMInterpretCLTS(pVM, pVCpu);
11014 AssertRCReturn(rc, rc);
11015 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11016 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
11017 Log4(("CRX CLTS write rc=%d\n", rc));
11018 break;
11019 }
11020
11021 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
11022 {
11023 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11024 AssertRCReturn(rc, rc);
11025 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
11026 if (RT_LIKELY(rc == VINF_SUCCESS))
11027 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11028 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
11029 Log4(("CRX LMSW write rc=%d\n", rc));
11030 break;
11031 }
11032
11033 default:
11034 {
11035 AssertMsgFailed(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType));
11036 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11037 }
11038 }
11039
11040 /* Validate possible error codes. */
11041 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
11042 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
11043 if (RT_SUCCESS(rc))
11044 {
11045 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11046 AssertRCReturn(rc2, rc2);
11047 }
11048
11049 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
11050 return rc;
11051}
11052
11053
11054/**
11055 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
11056 * VM-exit.
11057 */
11058HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11059{
11060 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11061 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
11062
11063 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11064 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11065 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11066 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
11067 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
11068 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
11069 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
11070 AssertRCReturn(rc2, rc2);
11071
11072 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
11073 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
11074 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
11075 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
11076 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
11077 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
11078 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11079 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
11080
11081 /* I/O operation lookup arrays. */
11082 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
11083 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
11084
11085 VBOXSTRICTRC rcStrict;
11086 uint32_t const cbValue = s_aIOSizes[uIOWidth];
11087 uint32_t const cbInstr = pVmxTransient->cbInstr;
11088 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
11089 PVM pVM = pVCpu->CTX_SUFF(pVM);
11090 if (fIOString)
11091 {
11092#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
11093 /*
11094 * INS/OUTS - I/O String instruction.
11095 *
11096 * Use instruction-information if available, otherwise fall back on
11097 * interpreting the instruction.
11098 */
11099 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11100 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
11101 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
11102 {
11103 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11104 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11105 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11106 AssertRCReturn(rc2, rc2);
11107 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
11108 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
11109 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
11110 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
11111 if (fIOWrite)
11112 {
11113 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
11114 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
11115 }
11116 else
11117 {
11118 /*
11119 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
11120 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
11121 * See Intel Instruction spec. for "INS".
11122 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
11123 */
11124 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
11125 }
11126 }
11127 else
11128 {
11129 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11130 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11131 AssertRCReturn(rc2, rc2);
11132 rcStrict = IEMExecOne(pVCpu);
11133 }
11134 /** @todo IEM needs to be setting these flags somehow. */
11135 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11136 fUpdateRipAlready = true;
11137#else
11138 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11139 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
11140 if (RT_SUCCESS(rcStrict))
11141 {
11142 if (fIOWrite)
11143 {
11144 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11145 (DISCPUMODE)pDis->uAddrMode, cbValue);
11146 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
11147 }
11148 else
11149 {
11150 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11151 (DISCPUMODE)pDis->uAddrMode, cbValue);
11152 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11153 }
11154 }
11155 else
11156 {
11157 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
11158 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11159 }
11160#endif
11161 }
11162 else
11163 {
11164 /*
11165 * IN/OUT - I/O instruction.
11166 */
11167 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11168 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11169 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11170 if (fIOWrite)
11171 {
11172 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11173 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11174 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11175 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11176 }
11177 else
11178 {
11179 uint32_t u32Result = 0;
11180 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11181 if (IOM_SUCCESS(rcStrict))
11182 {
11183 /* Save result of I/O IN instr. in AL/AX/EAX. */
11184 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11185 }
11186 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11187 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11188 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11189 }
11190 }
11191
11192 if (IOM_SUCCESS(rcStrict))
11193 {
11194 if (!fUpdateRipAlready)
11195 {
11196 pMixedCtx->rip += cbInstr;
11197 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11198 }
11199
11200 /*
11201 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11202 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11203 */
11204 if (fIOString)
11205 {
11206 /** @todo Single-step for INS/OUTS with REP prefix? */
11207 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11208 }
11209 else if (fStepping)
11210 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11211
11212 /*
11213 * If any I/O breakpoints are armed, we need to check if one triggered
11214 * and take appropriate action.
11215 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11216 */
11217 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11218 AssertRCReturn(rc2, rc2);
11219
11220 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11221 * execution engines about whether hyper BPs and such are pending. */
11222 uint32_t const uDr7 = pMixedCtx->dr[7];
11223 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11224 && X86_DR7_ANY_RW_IO(uDr7)
11225 && (pMixedCtx->cr4 & X86_CR4_DE))
11226 || DBGFBpIsHwIoArmed(pVM)))
11227 {
11228 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11229
11230 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11231 VMMRZCallRing3Disable(pVCpu);
11232 HM_DISABLE_PREEMPT_IF_NEEDED();
11233
11234 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11235
11236 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11237 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11238 {
11239 /* Raise #DB. */
11240 if (fIsGuestDbgActive)
11241 ASMSetDR6(pMixedCtx->dr[6]);
11242 if (pMixedCtx->dr[7] != uDr7)
11243 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11244
11245 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11246 }
11247 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11248 else if ( rcStrict2 != VINF_SUCCESS
11249 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11250 rcStrict = rcStrict2;
11251
11252 HM_RESTORE_PREEMPT_IF_NEEDED();
11253 VMMRZCallRing3Enable(pVCpu);
11254 }
11255 }
11256
11257#ifdef DEBUG
11258 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11259 Assert(!fIOWrite);
11260 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11261 Assert(fIOWrite);
11262 else
11263 {
11264 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11265 * statuses, that the VMM device and some others may return. See
11266 * IOM_SUCCESS() for guidance. */
11267 AssertMsg( RT_FAILURE(rcStrict)
11268 || rcStrict == VINF_SUCCESS
11269 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11270 || rcStrict == VINF_EM_DBG_BREAKPOINT
11271 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11272 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11273 }
11274#endif
11275
11276 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11277 return VBOXSTRICTRC_TODO(rcStrict);
11278}
11279
11280
11281/**
11282 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11283 * VM-exit.
11284 */
11285HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11286{
11287 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11288
11289 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11290 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11291 AssertRCReturn(rc, rc);
11292 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11293 {
11294 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11295 AssertRCReturn(rc, rc);
11296 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11297 {
11298 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11299
11300 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11301 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11302
11303 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11304 Assert(!pVCpu->hm.s.Event.fPending);
11305 pVCpu->hm.s.Event.fPending = true;
11306 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11307 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11308 AssertRCReturn(rc, rc);
11309 if (fErrorCodeValid)
11310 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11311 else
11312 pVCpu->hm.s.Event.u32ErrCode = 0;
11313 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11314 && uVector == X86_XCPT_PF)
11315 {
11316 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11317 }
11318
11319 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11320 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11321 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11322 }
11323 }
11324
11325 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11326 * emulation. */
11327 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11328 return VERR_EM_INTERPRETER;
11329}
11330
11331
11332/**
11333 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11334 */
11335HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11336{
11337 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11338 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11339 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11340 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11341 AssertRCReturn(rc, rc);
11342 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11343 return VINF_EM_DBG_STEPPED;
11344}
11345
11346
11347/**
11348 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11349 */
11350HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11351{
11352 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11353
11354 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11355 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11356 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11357 {
11358 if (rc == VINF_HM_DOUBLE_FAULT)
11359 rc = VINF_SUCCESS;
11360 return rc;
11361 }
11362
11363#if 0
11364 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11365 * just sync the whole thing. */
11366 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11367#else
11368 /* Aggressive state sync. for now. */
11369 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11370 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11371 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11372#endif
11373 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11374 AssertRCReturn(rc, rc);
11375
11376 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11377 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11378 switch (uAccessType)
11379 {
11380 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11381 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11382 {
11383 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11384 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11385 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11386
11387 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11388 GCPhys &= PAGE_BASE_GC_MASK;
11389 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11390 PVM pVM = pVCpu->CTX_SUFF(pVM);
11391 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11392 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11393
11394 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11395 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11396 CPUMCTX2CORE(pMixedCtx), GCPhys);
11397 rc = VBOXSTRICTRC_VAL(rc2);
11398 Log4(("ApicAccess rc=%d\n", rc));
11399 if ( rc == VINF_SUCCESS
11400 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11401 || rc == VERR_PAGE_NOT_PRESENT)
11402 {
11403 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11404 | HM_CHANGED_GUEST_RSP
11405 | HM_CHANGED_GUEST_RFLAGS
11406 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11407 rc = VINF_SUCCESS;
11408 }
11409 break;
11410 }
11411
11412 default:
11413 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11414 rc = VINF_EM_RAW_EMULATE_INSTR;
11415 break;
11416 }
11417
11418 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11419 if (rc != VINF_SUCCESS)
11420 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
11421 return rc;
11422}
11423
11424
11425/**
11426 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11427 * VM-exit.
11428 */
11429HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11430{
11431 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11432
11433 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11434 if (pVmxTransient->fWasGuestDebugStateActive)
11435 {
11436 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11437 HMVMX_RETURN_UNEXPECTED_EXIT();
11438 }
11439
11440 int rc = VERR_INTERNAL_ERROR_5;
11441 if ( !DBGFIsStepping(pVCpu)
11442 && !pVCpu->hm.s.fSingleInstruction
11443 && !pVmxTransient->fWasHyperDebugStateActive)
11444 {
11445 /* Don't intercept MOV DRx and #DB any more. */
11446 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11447 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11448 AssertRCReturn(rc, rc);
11449
11450 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11451 {
11452#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11453 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11454 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
11455#endif
11456 }
11457
11458 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11459 VMMRZCallRing3Disable(pVCpu);
11460 HM_DISABLE_PREEMPT_IF_NEEDED();
11461
11462 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11463 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11464 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11465
11466 HM_RESTORE_PREEMPT_IF_NEEDED();
11467 VMMRZCallRing3Enable(pVCpu);
11468
11469#ifdef VBOX_WITH_STATISTICS
11470 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11471 AssertRCReturn(rc, rc);
11472 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11473 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11474 else
11475 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11476#endif
11477 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11478 return VINF_SUCCESS;
11479 }
11480
11481 /*
11482 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11483 * Update the segment registers and DR7 from the CPU.
11484 */
11485 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11486 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11487 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11488 AssertRCReturn(rc, rc);
11489 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11490
11491 PVM pVM = pVCpu->CTX_SUFF(pVM);
11492 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11493 {
11494 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11495 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11496 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11497 if (RT_SUCCESS(rc))
11498 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11499 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11500 }
11501 else
11502 {
11503 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11504 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11505 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11506 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11507 }
11508
11509 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11510 if (RT_SUCCESS(rc))
11511 {
11512 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11513 AssertRCReturn(rc2, rc2);
11514 }
11515 return rc;
11516}
11517
11518
11519/**
11520 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11521 * Conditional VM-exit.
11522 */
11523HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11524{
11525 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11526 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11527
11528 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11529 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11530 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11531 {
11532 if (rc == VINF_HM_DOUBLE_FAULT)
11533 rc = VINF_SUCCESS;
11534 return rc;
11535 }
11536
11537 RTGCPHYS GCPhys = 0;
11538 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11539
11540#if 0
11541 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11542#else
11543 /* Aggressive state sync. for now. */
11544 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11545 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11546 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11547#endif
11548 AssertRCReturn(rc, rc);
11549
11550 /*
11551 * If we succeed, resume guest execution.
11552 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11553 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11554 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11555 * weird case. See @bugref{6043}.
11556 */
11557 PVM pVM = pVCpu->CTX_SUFF(pVM);
11558 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11559 rc = VBOXSTRICTRC_VAL(rc2);
11560 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11561 if ( rc == VINF_SUCCESS
11562 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11563 || rc == VERR_PAGE_NOT_PRESENT)
11564 {
11565 /* Successfully handled MMIO operation. */
11566 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11567 | HM_CHANGED_GUEST_RSP
11568 | HM_CHANGED_GUEST_RFLAGS
11569 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11570 rc = VINF_SUCCESS;
11571 }
11572 return rc;
11573}
11574
11575
11576/**
11577 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11578 * VM-exit.
11579 */
11580HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11581{
11582 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11583 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11584
11585 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11586 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11587 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11588 {
11589 if (rc == VINF_HM_DOUBLE_FAULT)
11590 rc = VINF_SUCCESS;
11591 return rc;
11592 }
11593
11594 RTGCPHYS GCPhys = 0;
11595 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11596 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11597#if 0
11598 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11599#else
11600 /* Aggressive state sync. for now. */
11601 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11602 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11603 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11604#endif
11605 AssertRCReturn(rc, rc);
11606
11607 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11608 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11609
11610 RTGCUINT uErrorCode = 0;
11611 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11612 uErrorCode |= X86_TRAP_PF_ID;
11613 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11614 uErrorCode |= X86_TRAP_PF_RW;
11615 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11616 uErrorCode |= X86_TRAP_PF_P;
11617
11618 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11619
11620 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11621 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11622
11623 /* Handle the pagefault trap for the nested shadow table. */
11624 PVM pVM = pVCpu->CTX_SUFF(pVM);
11625 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11626 TRPMResetTrap(pVCpu);
11627
11628 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11629 if ( rc == VINF_SUCCESS
11630 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11631 || rc == VERR_PAGE_NOT_PRESENT)
11632 {
11633 /* Successfully synced our nested page tables. */
11634 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11635 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11636 | HM_CHANGED_GUEST_RSP
11637 | HM_CHANGED_GUEST_RFLAGS);
11638 return VINF_SUCCESS;
11639 }
11640
11641 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11642 return rc;
11643}
11644
11645/** @} */
11646
11647/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11648/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11649/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11650
11651/** @name VM-exit exception handlers.
11652 * @{
11653 */
11654
11655/**
11656 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11657 */
11658static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11659{
11660 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11661 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11662
11663 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11664 AssertRCReturn(rc, rc);
11665
11666 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11667 {
11668 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11669 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11670
11671 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11672 * provides VM-exit instruction length. If this causes problem later,
11673 * disassemble the instruction like it's done on AMD-V. */
11674 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11675 AssertRCReturn(rc2, rc2);
11676 return rc;
11677 }
11678
11679 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11680 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11681 return rc;
11682}
11683
11684
11685/**
11686 * VM-exit exception handler for #BP (Breakpoint exception).
11687 */
11688static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11689{
11690 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11691 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11692
11693 /** @todo Try optimize this by not saving the entire guest state unless
11694 * really needed. */
11695 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11696 AssertRCReturn(rc, rc);
11697
11698 PVM pVM = pVCpu->CTX_SUFF(pVM);
11699 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11700 if (rc == VINF_EM_RAW_GUEST_TRAP)
11701 {
11702 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11703 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11704 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11705 AssertRCReturn(rc, rc);
11706
11707 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11708 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11709 }
11710
11711 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11712 return rc;
11713}
11714
11715
11716/**
11717 * VM-exit exception handler for #DB (Debug exception).
11718 */
11719static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11720{
11721 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11722 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11723 Log6(("XcptDB\n"));
11724
11725 /*
11726 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11727 * for processing.
11728 */
11729 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11730 AssertRCReturn(rc, rc);
11731
11732 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11733 uint64_t uDR6 = X86_DR6_INIT_VAL;
11734 uDR6 |= ( pVmxTransient->uExitQualification
11735 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11736
11737 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11738 if (rc == VINF_EM_RAW_GUEST_TRAP)
11739 {
11740 /*
11741 * The exception was for the guest. Update DR6, DR7.GD and
11742 * IA32_DEBUGCTL.LBR before forwarding it.
11743 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11744 */
11745 VMMRZCallRing3Disable(pVCpu);
11746 HM_DISABLE_PREEMPT_IF_NEEDED();
11747
11748 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11749 pMixedCtx->dr[6] |= uDR6;
11750 if (CPUMIsGuestDebugStateActive(pVCpu))
11751 ASMSetDR6(pMixedCtx->dr[6]);
11752
11753 HM_RESTORE_PREEMPT_IF_NEEDED();
11754 VMMRZCallRing3Enable(pVCpu);
11755
11756 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11757 AssertRCReturn(rc, rc);
11758
11759 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11760 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11761
11762 /* Paranoia. */
11763 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11764 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11765
11766 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11767 AssertRCReturn(rc, rc);
11768
11769 /*
11770 * Raise #DB in the guest.
11771 *
11772 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11773 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11774 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11775 *
11776 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11777 */
11778 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11779 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11780 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11781 AssertRCReturn(rc, rc);
11782 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11783 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11784 return VINF_SUCCESS;
11785 }
11786
11787 /*
11788 * Not a guest trap, must be a hypervisor related debug event then.
11789 * Update DR6 in case someone is interested in it.
11790 */
11791 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11792 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11793 CPUMSetHyperDR6(pVCpu, uDR6);
11794
11795 return rc;
11796}
11797
11798
11799/**
11800 * VM-exit exception handler for #NM (Device-not-available exception: floating
11801 * point exception).
11802 */
11803static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11804{
11805 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11806
11807 /* We require CR0 and EFER. EFER is always up-to-date. */
11808 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11809 AssertRCReturn(rc, rc);
11810
11811 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11812 VMMRZCallRing3Disable(pVCpu);
11813 HM_DISABLE_PREEMPT_IF_NEEDED();
11814
11815 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11816 if (pVmxTransient->fWasGuestFPUStateActive)
11817 {
11818 rc = VINF_EM_RAW_GUEST_TRAP;
11819 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11820 }
11821 else
11822 {
11823#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11824 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11825#endif
11826 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11827 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11828 }
11829
11830 HM_RESTORE_PREEMPT_IF_NEEDED();
11831 VMMRZCallRing3Enable(pVCpu);
11832
11833 if (rc == VINF_SUCCESS)
11834 {
11835 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11836 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11837 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11838 pVCpu->hm.s.fPreloadGuestFpu = true;
11839 }
11840 else
11841 {
11842 /* Forward #NM to the guest. */
11843 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11844 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11845 AssertRCReturn(rc, rc);
11846 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11847 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11848 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11849 }
11850
11851 return VINF_SUCCESS;
11852}
11853
11854
11855/**
11856 * VM-exit exception handler for #GP (General-protection exception).
11857 *
11858 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11859 */
11860static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11861{
11862 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11863 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11864
11865 int rc = VERR_INTERNAL_ERROR_5;
11866 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11867 {
11868#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11869 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11870 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11871 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11872 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11873 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11874 AssertRCReturn(rc, rc);
11875 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11876 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11877 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11878 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11879 return rc;
11880#else
11881 /* We don't intercept #GP. */
11882 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11883 NOREF(pVmxTransient);
11884 return VERR_VMX_UNEXPECTED_EXCEPTION;
11885#endif
11886 }
11887
11888 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11889 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11890
11891 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11892 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11893 AssertRCReturn(rc, rc);
11894
11895 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11896 uint32_t cbOp = 0;
11897 PVM pVM = pVCpu->CTX_SUFF(pVM);
11898 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11899 if (RT_SUCCESS(rc))
11900 {
11901 rc = VINF_SUCCESS;
11902 Assert(cbOp == pDis->cbInstr);
11903 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11904 switch (pDis->pCurInstr->uOpcode)
11905 {
11906 case OP_CLI:
11907 {
11908 pMixedCtx->eflags.Bits.u1IF = 0;
11909 pMixedCtx->eflags.Bits.u1RF = 0;
11910 pMixedCtx->rip += pDis->cbInstr;
11911 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11912 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11913 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11914 break;
11915 }
11916
11917 case OP_STI:
11918 {
11919 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11920 pMixedCtx->eflags.Bits.u1IF = 1;
11921 pMixedCtx->eflags.Bits.u1RF = 0;
11922 pMixedCtx->rip += pDis->cbInstr;
11923 if (!fOldIF)
11924 {
11925 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11926 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11927 }
11928 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11929 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11930 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11931 break;
11932 }
11933
11934 case OP_HLT:
11935 {
11936 rc = VINF_EM_HALT;
11937 pMixedCtx->rip += pDis->cbInstr;
11938 pMixedCtx->eflags.Bits.u1RF = 0;
11939 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11940 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11941 break;
11942 }
11943
11944 case OP_POPF:
11945 {
11946 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11947 uint32_t cbParm;
11948 uint32_t uMask;
11949 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11950 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11951 {
11952 cbParm = 4;
11953 uMask = 0xffffffff;
11954 }
11955 else
11956 {
11957 cbParm = 2;
11958 uMask = 0xffff;
11959 }
11960
11961 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11962 RTGCPTR GCPtrStack = 0;
11963 X86EFLAGS Eflags;
11964 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11965 &GCPtrStack);
11966 if (RT_SUCCESS(rc))
11967 {
11968 Assert(sizeof(Eflags.u32) >= cbParm);
11969 Eflags.u32 = 0;
11970 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11971 }
11972 if (RT_FAILURE(rc))
11973 {
11974 rc = VERR_EM_INTERPRETER;
11975 break;
11976 }
11977 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11978 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
11979 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11980 pMixedCtx->esp += cbParm;
11981 pMixedCtx->esp &= uMask;
11982 pMixedCtx->rip += pDis->cbInstr;
11983 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11984 | HM_CHANGED_GUEST_RSP
11985 | HM_CHANGED_GUEST_RFLAGS);
11986 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11987 if (fStepping)
11988 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11989
11990 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11991 break;
11992 }
11993
11994 case OP_PUSHF:
11995 {
11996 uint32_t cbParm;
11997 uint32_t uMask;
11998 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11999 {
12000 cbParm = 4;
12001 uMask = 0xffffffff;
12002 }
12003 else
12004 {
12005 cbParm = 2;
12006 uMask = 0xffff;
12007 }
12008
12009 /* Get the stack pointer & push the contents of eflags onto the stack. */
12010 RTGCPTR GCPtrStack = 0;
12011 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
12012 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
12013 if (RT_FAILURE(rc))
12014 {
12015 rc = VERR_EM_INTERPRETER;
12016 break;
12017 }
12018 X86EFLAGS Eflags = pMixedCtx->eflags;
12019 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
12020 Eflags.Bits.u1RF = 0;
12021 Eflags.Bits.u1VM = 0;
12022
12023 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
12024 if (RT_FAILURE(rc))
12025 {
12026 rc = VERR_EM_INTERPRETER;
12027 break;
12028 }
12029 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
12030 pMixedCtx->esp -= cbParm;
12031 pMixedCtx->esp &= uMask;
12032 pMixedCtx->rip += pDis->cbInstr;
12033 pMixedCtx->eflags.Bits.u1RF = 0;
12034 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12035 | HM_CHANGED_GUEST_RSP
12036 | HM_CHANGED_GUEST_RFLAGS);
12037 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
12038 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
12039 break;
12040 }
12041
12042 case OP_IRET:
12043 {
12044 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
12045 * instruction reference. */
12046 RTGCPTR GCPtrStack = 0;
12047 uint32_t uMask = 0xffff;
12048 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12049 uint16_t aIretFrame[3];
12050 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
12051 {
12052 rc = VERR_EM_INTERPRETER;
12053 break;
12054 }
12055 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12056 &GCPtrStack);
12057 if (RT_SUCCESS(rc))
12058 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
12059 if (RT_FAILURE(rc))
12060 {
12061 rc = VERR_EM_INTERPRETER;
12062 break;
12063 }
12064 pMixedCtx->eip = 0;
12065 pMixedCtx->ip = aIretFrame[0];
12066 pMixedCtx->cs.Sel = aIretFrame[1];
12067 pMixedCtx->cs.ValidSel = aIretFrame[1];
12068 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
12069 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
12070 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
12071 pMixedCtx->sp += sizeof(aIretFrame);
12072 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12073 | HM_CHANGED_GUEST_SEGMENT_REGS
12074 | HM_CHANGED_GUEST_RSP
12075 | HM_CHANGED_GUEST_RFLAGS);
12076 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
12077 if (fStepping)
12078 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
12079 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
12080 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
12081 break;
12082 }
12083
12084 case OP_INT:
12085 {
12086 uint16_t uVector = pDis->Param1.uValue & 0xff;
12087 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
12088 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12089 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12090 break;
12091 }
12092
12093 case OP_INTO:
12094 {
12095 if (pMixedCtx->eflags.Bits.u1OF)
12096 {
12097 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
12098 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12099 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12100 }
12101 else
12102 {
12103 pMixedCtx->eflags.Bits.u1RF = 0;
12104 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12105 }
12106 break;
12107 }
12108
12109 default:
12110 {
12111 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
12112 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
12113 EMCODETYPE_SUPERVISOR);
12114 rc = VBOXSTRICTRC_VAL(rc2);
12115 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
12116 /** @todo We have to set pending-debug exceptions here when the guest is
12117 * single-stepping depending on the instruction that was interpreted. */
12118 Log4(("#GP rc=%Rrc\n", rc));
12119 break;
12120 }
12121 }
12122 }
12123 else
12124 rc = VERR_EM_INTERPRETER;
12125
12126 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
12127 ("#GP Unexpected rc=%Rrc\n", rc));
12128 return rc;
12129}
12130
12131
12132#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12133/**
12134 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
12135 * the exception reported in the VMX transient structure back into the VM.
12136 *
12137 * @remarks Requires uExitIntInfo in the VMX transient structure to be
12138 * up-to-date.
12139 */
12140static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12141{
12142 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12143
12144 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12145 hmR0VmxCheckExitDueToEventDelivery(). */
12146 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12147 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12148 AssertRCReturn(rc, rc);
12149 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12150
12151#ifdef DEBUG_ramshankar
12152 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12153 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12154 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12155#endif
12156
12157 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12158 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12159 return VINF_SUCCESS;
12160}
12161#endif
12162
12163
12164/**
12165 * VM-exit exception handler for #PF (Page-fault exception).
12166 */
12167static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12168{
12169 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12170 PVM pVM = pVCpu->CTX_SUFF(pVM);
12171 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12172 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12173 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12174 AssertRCReturn(rc, rc);
12175
12176#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12177 if (pVM->hm.s.fNestedPaging)
12178 {
12179 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12180 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12181 {
12182 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12183 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12184 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12185 }
12186 else
12187 {
12188 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12189 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12190 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12191 }
12192 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12193 return rc;
12194 }
12195#else
12196 Assert(!pVM->hm.s.fNestedPaging);
12197 NOREF(pVM);
12198#endif
12199
12200 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12201 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12202 if (pVmxTransient->fVectoringPF)
12203 {
12204 Assert(pVCpu->hm.s.Event.fPending);
12205 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12206 }
12207
12208 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12209 AssertRCReturn(rc, rc);
12210
12211 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12212 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12213
12214 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12215 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12216 (RTGCPTR)pVmxTransient->uExitQualification);
12217
12218 Log4(("#PF: rc=%Rrc\n", rc));
12219 if (rc == VINF_SUCCESS)
12220 {
12221 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12222 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12223 * memory? We don't update the whole state here... */
12224 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12225 | HM_CHANGED_GUEST_RSP
12226 | HM_CHANGED_GUEST_RFLAGS
12227 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12228 TRPMResetTrap(pVCpu);
12229 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12230 return rc;
12231 }
12232
12233 if (rc == VINF_EM_RAW_GUEST_TRAP)
12234 {
12235 if (!pVmxTransient->fVectoringDoublePF)
12236 {
12237 /* It's a guest page fault and needs to be reflected to the guest. */
12238 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12239 TRPMResetTrap(pVCpu);
12240 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12241 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12242 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12243 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12244 }
12245 else
12246 {
12247 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12248 TRPMResetTrap(pVCpu);
12249 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12250 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12251 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12252 }
12253
12254 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12255 return VINF_SUCCESS;
12256 }
12257
12258 TRPMResetTrap(pVCpu);
12259 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12260 return rc;
12261}
12262
12263/** @} */
12264
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette