VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 54075

Last change on this file since 54075 was 54075, checked in by vboxsync, 10 years ago

VMM/HMVMXR0: Fixed unintentional losing #DB intercepts while loading guest CR0.
Also removed #MF from the real-on-v86 mode exception mask, as it too like #NM is handled separately.
Improved some comments.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 506.4 KB
Line 
1/* $Id: HMVMXR0.cpp 54075 2015-02-04 16:40:35Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39# include "dtrace/VBoxVMM.h"
40
41#ifdef DEBUG_ramshankar
42# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
43# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
44# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_CHECK_GUEST_STATE
46# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
47# define HMVMX_ALWAYS_TRAP_PF
48# define HMVMX_ALWAYS_SWAP_FPU_STATE
49# define HMVMX_ALWAYS_FLUSH_TLB
50# define HMVMX_ALWAYS_SWAP_EFER
51#endif
52
53
54/*******************************************************************************
55* Defined Constants And Macros *
56*******************************************************************************/
57#if defined(RT_ARCH_AMD64)
58# define HMVMX_IS_64BIT_HOST_MODE() (true)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
61extern "C" uint32_t g_fVMXIs64bitHost;
62# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
63typedef uint64_t HMVMXHCUINTREG;
64#else
65# define HMVMX_IS_64BIT_HOST_MODE() (false)
66typedef RTHCUINTREG HMVMXHCUINTREG;
67#endif
68
69/** Use the function table. */
70#define HMVMX_USE_FUNCTION_TABLE
71
72/** Determine which tagged-TLB flush handler to use. */
73#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
74#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
75#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
76#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
77
78/** @name Updated-guest-state flags.
79 * @{ */
80#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
81#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
82#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
83#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
84#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
85#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
86#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
87#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
88#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
89#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
90#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
91#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
92#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
93#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
94#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
95#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
96#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
97#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
98#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
99#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
100#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
101 | HMVMX_UPDATED_GUEST_RSP \
102 | HMVMX_UPDATED_GUEST_RFLAGS \
103 | HMVMX_UPDATED_GUEST_CR0 \
104 | HMVMX_UPDATED_GUEST_CR3 \
105 | HMVMX_UPDATED_GUEST_CR4 \
106 | HMVMX_UPDATED_GUEST_GDTR \
107 | HMVMX_UPDATED_GUEST_IDTR \
108 | HMVMX_UPDATED_GUEST_LDTR \
109 | HMVMX_UPDATED_GUEST_TR \
110 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
111 | HMVMX_UPDATED_GUEST_DEBUG \
112 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
113 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
114 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
115 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
116 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
117 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
118 | HMVMX_UPDATED_GUEST_INTR_STATE \
119 | HMVMX_UPDATED_GUEST_APIC_STATE)
120/** @} */
121
122/** @name
123 * Flags to skip redundant reads of some common VMCS fields that are not part of
124 * the guest-CPU state but are in the transient structure.
125 */
126#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
127#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
131#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
132#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
133/** @} */
134
135/** @name
136 * States of the VMCS.
137 *
138 * This does not reflect all possible VMCS states but currently only those
139 * needed for maintaining the VMCS consistently even when thread-context hooks
140 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
141 */
142#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
143#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
144#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
145/** @} */
146
147/**
148 * Exception bitmap mask for real-mode guests (real-on-v86).
149 *
150 * We need to intercept all exceptions manually except:
151 * - #NM, #MF handled separately, see hmR0VmxLoadSharedCR0().
152 * - #PF need not be intercepted even in real-mode if we have Nested Paging
153 * support.
154 */
155#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
156 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
157 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
158 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
159 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
160 /* RT_BIT(X86_XCPT_MF) */ | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
161 | RT_BIT(X86_XCPT_XF))
162
163/**
164 * Exception bitmap mask for all contributory exceptions.
165 *
166 * Page fault is deliberately excluded here as it's conditional as to whether
167 * it's contributory or benign. Page faults are handled separately.
168 */
169#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
170 | RT_BIT(X86_XCPT_DE))
171
172/** Maximum VM-instruction error number. */
173#define HMVMX_INSTR_ERROR_MAX 28
174
175/** Profiling macro. */
176#ifdef HM_PROFILE_EXIT_DISPATCH
177# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
178# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
179#else
180# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
181# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
182#endif
183
184/** Assert that preemption is disabled or covered by thread-context hooks. */
185#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
186 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
187
188/** Assert that we haven't migrated CPUs when thread-context hooks are not
189 * used. */
190#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
191 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
192 ("Illegal migration! Entered on CPU %u Current %u\n", \
193 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
194
195/** Helper macro for VM-exit handlers called unexpectedly. */
196#define HMVMX_RETURN_UNEXPECTED_EXIT() \
197 do { \
198 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
199 return VERR_VMX_UNEXPECTED_EXIT; \
200 } while (0)
201
202
203/*******************************************************************************
204* Structures and Typedefs *
205*******************************************************************************/
206/**
207 * VMX transient state.
208 *
209 * A state structure for holding miscellaneous information across
210 * VMX non-root operation and restored after the transition.
211 */
212typedef struct VMXTRANSIENT
213{
214 /** The host's rflags/eflags. */
215 RTCCUINTREG uEflags;
216#if HC_ARCH_BITS == 32
217 uint32_t u32Alignment0;
218#endif
219 /** The guest's TPR value used for TPR shadowing. */
220 uint8_t u8GuestTpr;
221 /** Alignment. */
222 uint8_t abAlignment0[7];
223
224 /** The basic VM-exit reason. */
225 uint16_t uExitReason;
226 /** Alignment. */
227 uint16_t u16Alignment0;
228 /** The VM-exit interruption error code. */
229 uint32_t uExitIntErrorCode;
230 /** The VM-exit exit code qualification. */
231 uint64_t uExitQualification;
232
233 /** The VM-exit interruption-information field. */
234 uint32_t uExitIntInfo;
235 /** The VM-exit instruction-length field. */
236 uint32_t cbInstr;
237 /** The VM-exit instruction-information field. */
238 union
239 {
240 /** Plain unsigned int representation. */
241 uint32_t u;
242 /** INS and OUTS information. */
243 struct
244 {
245 uint32_t u6Reserved0 : 7;
246 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
247 uint32_t u3AddrSize : 3;
248 uint32_t u5Reserved1 : 5;
249 /** The segment register (X86_SREG_XXX). */
250 uint32_t iSegReg : 3;
251 uint32_t uReserved2 : 14;
252 } StrIo;
253 } ExitInstrInfo;
254 /** Whether the VM-entry failed or not. */
255 bool fVMEntryFailed;
256 /** Alignment. */
257 uint8_t abAlignment1[3];
258
259 /** The VM-entry interruption-information field. */
260 uint32_t uEntryIntInfo;
261 /** The VM-entry exception error code field. */
262 uint32_t uEntryXcptErrorCode;
263 /** The VM-entry instruction length field. */
264 uint32_t cbEntryInstr;
265
266 /** IDT-vectoring information field. */
267 uint32_t uIdtVectoringInfo;
268 /** IDT-vectoring error code. */
269 uint32_t uIdtVectoringErrorCode;
270
271 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
272 uint32_t fVmcsFieldsRead;
273
274 /** Whether the guest FPU was active at the time of VM-exit. */
275 bool fWasGuestFPUStateActive;
276 /** Whether the guest debug state was active at the time of VM-exit. */
277 bool fWasGuestDebugStateActive;
278 /** Whether the hyper debug state was active at the time of VM-exit. */
279 bool fWasHyperDebugStateActive;
280 /** Whether TSC-offsetting should be setup before VM-entry. */
281 bool fUpdateTscOffsettingAndPreemptTimer;
282 /** Whether the VM-exit was caused by a page-fault during delivery of a
283 * contributory exception or a page-fault. */
284 bool fVectoringDoublePF;
285 /** Whether the VM-exit was caused by a page-fault during delivery of an
286 * external interrupt or NMI. */
287 bool fVectoringPF;
288} VMXTRANSIENT;
289AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
290AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
291AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
292AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
293AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
294/** Pointer to VMX transient state. */
295typedef VMXTRANSIENT *PVMXTRANSIENT;
296
297
298/**
299 * MSR-bitmap read permissions.
300 */
301typedef enum VMXMSREXITREAD
302{
303 /** Reading this MSR causes a VM-exit. */
304 VMXMSREXIT_INTERCEPT_READ = 0xb,
305 /** Reading this MSR does not cause a VM-exit. */
306 VMXMSREXIT_PASSTHRU_READ
307} VMXMSREXITREAD;
308/** Pointer to MSR-bitmap read permissions. */
309typedef VMXMSREXITREAD* PVMXMSREXITREAD;
310
311/**
312 * MSR-bitmap write permissions.
313 */
314typedef enum VMXMSREXITWRITE
315{
316 /** Writing to this MSR causes a VM-exit. */
317 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
318 /** Writing to this MSR does not cause a VM-exit. */
319 VMXMSREXIT_PASSTHRU_WRITE
320} VMXMSREXITWRITE;
321/** Pointer to MSR-bitmap write permissions. */
322typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
323
324
325/**
326 * VMX VM-exit handler.
327 *
328 * @returns VBox status code.
329 * @param pVCpu Pointer to the VMCPU.
330 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
331 * out-of-sync. Make sure to update the required
332 * fields before using them.
333 * @param pVmxTransient Pointer to the VMX-transient structure.
334 */
335#ifndef HMVMX_USE_FUNCTION_TABLE
336typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
337#else
338typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
339/** Pointer to VM-exit handler. */
340typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
341#endif
342
343
344/*******************************************************************************
345* Internal Functions *
346*******************************************************************************/
347static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
348static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
349static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
350 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
351 bool fStepping, uint32_t *puIntState);
352#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
353static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
354#endif
355#ifndef HMVMX_USE_FUNCTION_TABLE
356DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
357# define HMVMX_EXIT_DECL static int
358#else
359# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
360#endif
361DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
362 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
363
364/** @name VM-exit handlers.
365 * @{
366 */
367static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
368static FNVMXEXITHANDLER hmR0VmxExitExtInt;
369static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
370static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
371static FNVMXEXITHANDLER hmR0VmxExitSipi;
372static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
373static FNVMXEXITHANDLER hmR0VmxExitSmi;
374static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
375static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
376static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
377static FNVMXEXITHANDLER hmR0VmxExitCpuid;
378static FNVMXEXITHANDLER hmR0VmxExitGetsec;
379static FNVMXEXITHANDLER hmR0VmxExitHlt;
380static FNVMXEXITHANDLER hmR0VmxExitInvd;
381static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
382static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
383static FNVMXEXITHANDLER hmR0VmxExitVmcall;
384static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
385static FNVMXEXITHANDLER hmR0VmxExitRsm;
386static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
387static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
388static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
389static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
390static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
391static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
392static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
393static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
394static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
395static FNVMXEXITHANDLER hmR0VmxExitMwait;
396static FNVMXEXITHANDLER hmR0VmxExitMtf;
397static FNVMXEXITHANDLER hmR0VmxExitMonitor;
398static FNVMXEXITHANDLER hmR0VmxExitPause;
399static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
400static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
401static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
402static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
403static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
404static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
405static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
406static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
407static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
408static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
409static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
410static FNVMXEXITHANDLER hmR0VmxExitRdrand;
411static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
412/** @} */
413
414static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
415static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
417static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
418static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
419static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
420#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
421static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
422#endif
423static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
424
425/*******************************************************************************
426* Global Variables *
427*******************************************************************************/
428#ifdef HMVMX_USE_FUNCTION_TABLE
429
430/**
431 * VMX_EXIT dispatch table.
432 */
433static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
434{
435 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
436 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
437 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
438 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
439 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
440 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
441 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
442 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
443 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
444 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
445 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
446 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
447 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
448 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
449 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
450 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
451 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
452 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
453 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
454 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
455 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
456 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
457 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
458 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
459 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
460 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
461 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
462 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
463 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
464 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
465 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
466 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
467 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
468 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
469 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
470 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
471 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
472 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
473 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
474 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
475 /* 40 UNDEFINED */ hmR0VmxExitPause,
476 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
477 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
478 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
479 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
480 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
481 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
482 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
483 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
484 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
485 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
486 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
487 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
488 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
489 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
490 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
491 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
493 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
494 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
495};
496#endif /* HMVMX_USE_FUNCTION_TABLE */
497
498#ifdef VBOX_STRICT
499static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
500{
501 /* 0 */ "(Not Used)",
502 /* 1 */ "VMCALL executed in VMX root operation.",
503 /* 2 */ "VMCLEAR with invalid physical address.",
504 /* 3 */ "VMCLEAR with VMXON pointer.",
505 /* 4 */ "VMLAUNCH with non-clear VMCS.",
506 /* 5 */ "VMRESUME with non-launched VMCS.",
507 /* 6 */ "VMRESUME after VMXOFF",
508 /* 7 */ "VM-entry with invalid control fields.",
509 /* 8 */ "VM-entry with invalid host state fields.",
510 /* 9 */ "VMPTRLD with invalid physical address.",
511 /* 10 */ "VMPTRLD with VMXON pointer.",
512 /* 11 */ "VMPTRLD with incorrect revision identifier.",
513 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
514 /* 13 */ "VMWRITE to read-only VMCS component.",
515 /* 14 */ "(Not Used)",
516 /* 15 */ "VMXON executed in VMX root operation.",
517 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
518 /* 17 */ "VM-entry with non-launched executing VMCS.",
519 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
520 /* 19 */ "VMCALL with non-clear VMCS.",
521 /* 20 */ "VMCALL with invalid VM-exit control fields.",
522 /* 21 */ "(Not Used)",
523 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
524 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
525 /* 24 */ "VMCALL with invalid SMM-monitor features.",
526 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
527 /* 26 */ "VM-entry with events blocked by MOV SS.",
528 /* 27 */ "(Not Used)",
529 /* 28 */ "Invalid operand to INVEPT/INVVPID."
530};
531#endif /* VBOX_STRICT */
532
533
534
535/**
536 * Updates the VM's last error record. If there was a VMX instruction error,
537 * reads the error data from the VMCS and updates VCPU's last error record as
538 * well.
539 *
540 * @param pVM Pointer to the VM.
541 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
542 * VERR_VMX_UNABLE_TO_START_VM or
543 * VERR_VMX_INVALID_VMCS_FIELD).
544 * @param rc The error code.
545 */
546static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
547{
548 AssertPtr(pVM);
549 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
550 || rc == VERR_VMX_UNABLE_TO_START_VM)
551 {
552 AssertPtrReturnVoid(pVCpu);
553 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
554 }
555 pVM->hm.s.lLastError = rc;
556}
557
558
559/**
560 * Reads the VM-entry interruption-information field from the VMCS into the VMX
561 * transient structure.
562 *
563 * @returns VBox status code.
564 * @param pVmxTransient Pointer to the VMX transient structure.
565 *
566 * @remarks No-long-jump zone!!!
567 */
568DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
569{
570 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
571 AssertRCReturn(rc, rc);
572 return VINF_SUCCESS;
573}
574
575
576/**
577 * Reads the VM-entry exception error code field from the VMCS into
578 * the VMX transient structure.
579 *
580 * @returns VBox status code.
581 * @param pVmxTransient Pointer to the VMX transient structure.
582 *
583 * @remarks No-long-jump zone!!!
584 */
585DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
586{
587 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
588 AssertRCReturn(rc, rc);
589 return VINF_SUCCESS;
590}
591
592
593/**
594 * Reads the VM-entry exception error code field from the VMCS into
595 * the VMX transient structure.
596 *
597 * @returns VBox status code.
598 * @param pVmxTransient Pointer to the VMX transient structure.
599 *
600 * @remarks No-long-jump zone!!!
601 */
602DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
603{
604 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
605 AssertRCReturn(rc, rc);
606 return VINF_SUCCESS;
607}
608
609
610/**
611 * Reads the VM-exit interruption-information field from the VMCS into the VMX
612 * transient structure.
613 *
614 * @returns VBox status code.
615 * @param pVmxTransient Pointer to the VMX transient structure.
616 */
617DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
618{
619 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
620 {
621 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
622 AssertRCReturn(rc, rc);
623 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
624 }
625 return VINF_SUCCESS;
626}
627
628
629/**
630 * Reads the VM-exit interruption error code from the VMCS into the VMX
631 * transient structure.
632 *
633 * @returns VBox status code.
634 * @param pVmxTransient Pointer to the VMX transient structure.
635 */
636DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
637{
638 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
639 {
640 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
641 AssertRCReturn(rc, rc);
642 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
643 }
644 return VINF_SUCCESS;
645}
646
647
648/**
649 * Reads the VM-exit instruction length field from the VMCS into the VMX
650 * transient structure.
651 *
652 * @returns VBox status code.
653 * @param pVCpu Pointer to the VMCPU.
654 * @param pVmxTransient Pointer to the VMX transient structure.
655 */
656DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
657{
658 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
659 {
660 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
661 AssertRCReturn(rc, rc);
662 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
663 }
664 return VINF_SUCCESS;
665}
666
667
668/**
669 * Reads the VM-exit instruction-information field from the VMCS into
670 * the VMX transient structure.
671 *
672 * @returns VBox status code.
673 * @param pVmxTransient Pointer to the VMX transient structure.
674 */
675DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
676{
677 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
678 {
679 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
680 AssertRCReturn(rc, rc);
681 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
682 }
683 return VINF_SUCCESS;
684}
685
686
687/**
688 * Reads the exit code qualification from the VMCS into the VMX transient
689 * structure.
690 *
691 * @returns VBox status code.
692 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
693 * case).
694 * @param pVmxTransient Pointer to the VMX transient structure.
695 */
696DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
697{
698 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
699 {
700 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
701 AssertRCReturn(rc, rc);
702 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
703 }
704 return VINF_SUCCESS;
705}
706
707
708/**
709 * Reads the IDT-vectoring information field from the VMCS into the VMX
710 * transient structure.
711 *
712 * @returns VBox status code.
713 * @param pVmxTransient Pointer to the VMX transient structure.
714 *
715 * @remarks No-long-jump zone!!!
716 */
717DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
718{
719 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
720 {
721 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
722 AssertRCReturn(rc, rc);
723 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
724 }
725 return VINF_SUCCESS;
726}
727
728
729/**
730 * Reads the IDT-vectoring error code from the VMCS into the VMX
731 * transient structure.
732 *
733 * @returns VBox status code.
734 * @param pVmxTransient Pointer to the VMX transient structure.
735 */
736DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
737{
738 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
739 {
740 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
741 AssertRCReturn(rc, rc);
742 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
743 }
744 return VINF_SUCCESS;
745}
746
747
748/**
749 * Enters VMX root mode operation on the current CPU.
750 *
751 * @returns VBox status code.
752 * @param pVM Pointer to the VM (optional, can be NULL, after
753 * a resume).
754 * @param HCPhysCpuPage Physical address of the VMXON region.
755 * @param pvCpuPage Pointer to the VMXON region.
756 */
757static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
758{
759 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
760 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
761 Assert(pvCpuPage);
762 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
763
764 if (pVM)
765 {
766 /* Write the VMCS revision dword to the VMXON region. */
767 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
768 }
769
770 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
771 RTCCUINTREG uEflags = ASMIntDisableFlags();
772
773 /* Enable the VMX bit in CR4 if necessary. */
774 RTCCUINTREG uCr4 = ASMGetCR4();
775 if (!(uCr4 & X86_CR4_VMXE))
776 ASMSetCR4(uCr4 | X86_CR4_VMXE);
777
778 /* Enter VMX root mode. */
779 int rc = VMXEnable(HCPhysCpuPage);
780 if (RT_FAILURE(rc))
781 ASMSetCR4(uCr4);
782
783 /* Restore interrupts. */
784 ASMSetFlags(uEflags);
785 return rc;
786}
787
788
789/**
790 * Exits VMX root mode operation on the current CPU.
791 *
792 * @returns VBox status code.
793 */
794static int hmR0VmxLeaveRootMode(void)
795{
796 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
797
798 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
799 RTCCUINTREG uEflags = ASMIntDisableFlags();
800
801 /* If we're for some reason not in VMX root mode, then don't leave it. */
802 RTCCUINTREG uHostCR4 = ASMGetCR4();
803
804 int rc;
805 if (uHostCR4 & X86_CR4_VMXE)
806 {
807 /* Exit VMX root mode and clear the VMX bit in CR4. */
808 VMXDisable();
809 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
810 rc = VINF_SUCCESS;
811 }
812 else
813 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
814
815 /* Restore interrupts. */
816 ASMSetFlags(uEflags);
817 return rc;
818}
819
820
821/**
822 * Allocates and maps one physically contiguous page. The allocated page is
823 * zero'd out. (Used by various VT-x structures).
824 *
825 * @returns IPRT status code.
826 * @param pMemObj Pointer to the ring-0 memory object.
827 * @param ppVirt Where to store the virtual address of the
828 * allocation.
829 * @param pPhys Where to store the physical address of the
830 * allocation.
831 */
832DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
833{
834 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
835 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
836 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
837
838 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
839 if (RT_FAILURE(rc))
840 return rc;
841 *ppVirt = RTR0MemObjAddress(*pMemObj);
842 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
843 ASMMemZero32(*ppVirt, PAGE_SIZE);
844 return VINF_SUCCESS;
845}
846
847
848/**
849 * Frees and unmaps an allocated physical page.
850 *
851 * @param pMemObj Pointer to the ring-0 memory object.
852 * @param ppVirt Where to re-initialize the virtual address of
853 * allocation as 0.
854 * @param pHCPhys Where to re-initialize the physical address of the
855 * allocation as 0.
856 */
857DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
858{
859 AssertPtr(pMemObj);
860 AssertPtr(ppVirt);
861 AssertPtr(pHCPhys);
862 if (*pMemObj != NIL_RTR0MEMOBJ)
863 {
864 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
865 AssertRC(rc);
866 *pMemObj = NIL_RTR0MEMOBJ;
867 *ppVirt = 0;
868 *pHCPhys = 0;
869 }
870}
871
872
873/**
874 * Worker function to free VT-x related structures.
875 *
876 * @returns IPRT status code.
877 * @param pVM Pointer to the VM.
878 */
879static void hmR0VmxStructsFree(PVM pVM)
880{
881 for (VMCPUID i = 0; i < pVM->cCpus; i++)
882 {
883 PVMCPU pVCpu = &pVM->aCpus[i];
884 AssertPtr(pVCpu);
885
886 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
887 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
888
889 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
890 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
891
892 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
893 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
894 }
895
896 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
897#ifdef VBOX_WITH_CRASHDUMP_MAGIC
898 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
899#endif
900}
901
902
903/**
904 * Worker function to allocate VT-x related VM structures.
905 *
906 * @returns IPRT status code.
907 * @param pVM Pointer to the VM.
908 */
909static int hmR0VmxStructsAlloc(PVM pVM)
910{
911 /*
912 * Initialize members up-front so we can cleanup properly on allocation failure.
913 */
914#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
915 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
916 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
917 pVM->hm.s.vmx.HCPhys##a_Name = 0;
918
919#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
920 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
921 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
922 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
923
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
926#endif
927 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
928
929 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
930 for (VMCPUID i = 0; i < pVM->cCpus; i++)
931 {
932 PVMCPU pVCpu = &pVM->aCpus[i];
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
935 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
936 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
937 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
938 }
939#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
940#undef VMXLOCAL_INIT_VM_MEMOBJ
941
942 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
943 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
944 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
945 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
946
947 /*
948 * Allocate all the VT-x structures.
949 */
950 int rc = VINF_SUCCESS;
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
953 if (RT_FAILURE(rc))
954 goto cleanup;
955 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
956 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
957#endif
958
959 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
960 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
961 {
962 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
963 &pVM->hm.s.vmx.HCPhysApicAccess);
964 if (RT_FAILURE(rc))
965 goto cleanup;
966 }
967
968 /*
969 * Initialize per-VCPU VT-x structures.
970 */
971 for (VMCPUID i = 0; i < pVM->cCpus; i++)
972 {
973 PVMCPU pVCpu = &pVM->aCpus[i];
974 AssertPtr(pVCpu);
975
976 /* Allocate the VM control structure (VMCS). */
977 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
978 if (RT_FAILURE(rc))
979 goto cleanup;
980
981 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
982 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
983 {
984 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
985 &pVCpu->hm.s.vmx.HCPhysVirtApic);
986 if (RT_FAILURE(rc))
987 goto cleanup;
988 }
989
990 /*
991 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
992 * transparent accesses of specific MSRs.
993 *
994 * If the condition for enabling MSR bitmaps changes here, don't forget to
995 * update HMIsMsrBitmapsAvailable().
996 */
997 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
998 {
999 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1000 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1001 if (RT_FAILURE(rc))
1002 goto cleanup;
1003 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1004 }
1005
1006 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1007 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1008 if (RT_FAILURE(rc))
1009 goto cleanup;
1010
1011 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1012 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 return VINF_SUCCESS;
1018
1019cleanup:
1020 hmR0VmxStructsFree(pVM);
1021 return rc;
1022}
1023
1024
1025/**
1026 * Does global VT-x initialization (called during module initialization).
1027 *
1028 * @returns VBox status code.
1029 */
1030VMMR0DECL(int) VMXR0GlobalInit(void)
1031{
1032#ifdef HMVMX_USE_FUNCTION_TABLE
1033 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1034# ifdef VBOX_STRICT
1035 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1036 Assert(g_apfnVMExitHandlers[i]);
1037# endif
1038#endif
1039 return VINF_SUCCESS;
1040}
1041
1042
1043/**
1044 * Does global VT-x termination (called during module termination).
1045 */
1046VMMR0DECL(void) VMXR0GlobalTerm()
1047{
1048 /* Nothing to do currently. */
1049}
1050
1051
1052/**
1053 * Sets up and activates VT-x on the current CPU.
1054 *
1055 * @returns VBox status code.
1056 * @param pCpu Pointer to the global CPU info struct.
1057 * @param pVM Pointer to the VM (can be NULL after a host resume
1058 * operation).
1059 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1060 * fEnabledByHost is true).
1061 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1062 * @a fEnabledByHost is true).
1063 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1064 * enable VT-x on the host.
1065 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1066 */
1067VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1068 void *pvMsrs)
1069{
1070 Assert(pCpu);
1071 Assert(pvMsrs);
1072 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1073
1074 /* Enable VT-x if it's not already enabled by the host. */
1075 if (!fEnabledByHost)
1076 {
1077 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1078 if (RT_FAILURE(rc))
1079 return rc;
1080 }
1081
1082 /*
1083 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1084 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1085 */
1086 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1087 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1088 {
1089 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1090 pCpu->fFlushAsidBeforeUse = false;
1091 }
1092 else
1093 pCpu->fFlushAsidBeforeUse = true;
1094
1095 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1096 ++pCpu->cTlbFlushes;
1097
1098 return VINF_SUCCESS;
1099}
1100
1101
1102/**
1103 * Deactivates VT-x on the current CPU.
1104 *
1105 * @returns VBox status code.
1106 * @param pCpu Pointer to the global CPU info struct.
1107 * @param pvCpuPage Pointer to the VMXON region.
1108 * @param HCPhysCpuPage Physical address of the VMXON region.
1109 *
1110 * @remarks This function should never be called when SUPR0EnableVTx() or
1111 * similar was used to enable VT-x on the host.
1112 */
1113VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1114{
1115 NOREF(pCpu);
1116 NOREF(pvCpuPage);
1117 NOREF(HCPhysCpuPage);
1118
1119 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1120 return hmR0VmxLeaveRootMode();
1121}
1122
1123
1124/**
1125 * Sets the permission bits for the specified MSR in the MSR bitmap.
1126 *
1127 * @param pVCpu Pointer to the VMCPU.
1128 * @param uMSR The MSR value.
1129 * @param enmRead Whether reading this MSR causes a VM-exit.
1130 * @param enmWrite Whether writing this MSR causes a VM-exit.
1131 */
1132static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1133{
1134 int32_t iBit;
1135 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1136
1137 /*
1138 * Layout:
1139 * 0x000 - 0x3ff - Low MSR read bits
1140 * 0x400 - 0x7ff - High MSR read bits
1141 * 0x800 - 0xbff - Low MSR write bits
1142 * 0xc00 - 0xfff - High MSR write bits
1143 */
1144 if (uMsr <= 0x00001FFF)
1145 iBit = uMsr;
1146 else if ( uMsr >= 0xC0000000
1147 && uMsr <= 0xC0001FFF)
1148 {
1149 iBit = (uMsr - 0xC0000000);
1150 pbMsrBitmap += 0x400;
1151 }
1152 else
1153 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1154
1155 Assert(iBit <= 0x1fff);
1156 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1157 ASMBitSet(pbMsrBitmap, iBit);
1158 else
1159 ASMBitClear(pbMsrBitmap, iBit);
1160
1161 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1162 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1163 else
1164 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1165}
1166
1167
1168#ifdef VBOX_STRICT
1169/**
1170 * Gets the permission bits for the specified MSR in the MSR bitmap.
1171 *
1172 * @returns VBox status code.
1173 * @retval VINF_SUCCESS if the specified MSR is found.
1174 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1175 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1176 *
1177 * @param pVCpu Pointer to the VMCPU.
1178 * @param uMsr The MSR.
1179 * @param penmRead Where to store the read permissions.
1180 * @param penmWrite Where to store the write permissions.
1181 */
1182static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1183{
1184 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1185 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1186 int32_t iBit;
1187 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1188
1189 /* See hmR0VmxSetMsrPermission() for the layout. */
1190 if (uMsr <= 0x00001FFF)
1191 iBit = uMsr;
1192 else if ( uMsr >= 0xC0000000
1193 && uMsr <= 0xC0001FFF)
1194 {
1195 iBit = (uMsr - 0xC0000000);
1196 pbMsrBitmap += 0x400;
1197 }
1198 else
1199 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1200
1201 Assert(iBit <= 0x1fff);
1202 if (ASMBitTest(pbMsrBitmap, iBit))
1203 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1204 else
1205 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1206
1207 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1208 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1209 else
1210 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1211 return VINF_SUCCESS;
1212}
1213#endif /* VBOX_STRICT */
1214
1215
1216/**
1217 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1218 * area.
1219 *
1220 * @returns VBox status code.
1221 * @param pVCpu Pointer to the VMCPU.
1222 * @param cMsrs The number of MSRs.
1223 */
1224DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1225{
1226 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1227 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1228 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1229 {
1230 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1231 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1232 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1233 }
1234
1235 /* Update number of guest MSRs to load/store across the world-switch. */
1236 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1237 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1238
1239 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1240 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1241
1242 /* Update the VCPU's copy of the MSR count. */
1243 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1244
1245 return VINF_SUCCESS;
1246}
1247
1248
1249/**
1250 * Adds a new (or updates the value of an existing) guest/host MSR
1251 * pair to be swapped during the world-switch as part of the
1252 * auto-load/store MSR area in the VMCS.
1253 *
1254 * @returns true if the MSR was added -and- its value was updated, false
1255 * otherwise.
1256 * @param pVCpu Pointer to the VMCPU.
1257 * @param uMsr The MSR.
1258 * @param uGuestMsr Value of the guest MSR.
1259 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1260 * necessary.
1261 */
1262static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1263{
1264 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1265 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1266 uint32_t i;
1267 for (i = 0; i < cMsrs; i++)
1268 {
1269 if (pGuestMsr->u32Msr == uMsr)
1270 break;
1271 pGuestMsr++;
1272 }
1273
1274 bool fAdded = false;
1275 if (i == cMsrs)
1276 {
1277 ++cMsrs;
1278 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1279 AssertRC(rc);
1280
1281 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1282 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1283 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1284
1285 fAdded = true;
1286 }
1287
1288 /* Update the MSR values in the auto-load/store MSR area. */
1289 pGuestMsr->u32Msr = uMsr;
1290 pGuestMsr->u64Value = uGuestMsrValue;
1291
1292 /* Create/update the MSR slot in the host MSR area. */
1293 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1294 pHostMsr += i;
1295 pHostMsr->u32Msr = uMsr;
1296
1297 /*
1298 * Update the host MSR only when requested by the caller AND when we're
1299 * adding it to the auto-load/store area. Otherwise, it would have been
1300 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1301 */
1302 bool fUpdatedMsrValue = false;
1303 if ( fAdded
1304 && fUpdateHostMsr)
1305 {
1306 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1307 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1308 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1309 fUpdatedMsrValue = true;
1310 }
1311
1312 return fUpdatedMsrValue;
1313}
1314
1315
1316/**
1317 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1318 * auto-load/store MSR area in the VMCS.
1319 *
1320 * @returns VBox status code.
1321 * @param pVCpu Pointer to the VMCPU.
1322 * @param uMsr The MSR.
1323 */
1324static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1325{
1326 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1327 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1328 for (uint32_t i = 0; i < cMsrs; i++)
1329 {
1330 /* Find the MSR. */
1331 if (pGuestMsr->u32Msr == uMsr)
1332 {
1333 /* If it's the last MSR, simply reduce the count. */
1334 if (i == cMsrs - 1)
1335 {
1336 --cMsrs;
1337 break;
1338 }
1339
1340 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1341 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1342 pLastGuestMsr += cMsrs - 1;
1343 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1344 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1345
1346 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1347 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1348 pLastHostMsr += cMsrs - 1;
1349 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1350 pHostMsr->u64Value = pLastHostMsr->u64Value;
1351 --cMsrs;
1352 break;
1353 }
1354 pGuestMsr++;
1355 }
1356
1357 /* Update the VMCS if the count changed (meaning the MSR was found). */
1358 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1359 {
1360 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1361 AssertRCReturn(rc, rc);
1362
1363 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1364 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1365 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1366
1367 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1368 return VINF_SUCCESS;
1369 }
1370
1371 return VERR_NOT_FOUND;
1372}
1373
1374
1375/**
1376 * Checks if the specified guest MSR is part of the auto-load/store area in
1377 * the VMCS.
1378 *
1379 * @returns true if found, false otherwise.
1380 * @param pVCpu Pointer to the VMCPU.
1381 * @param uMsr The MSR to find.
1382 */
1383static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1384{
1385 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1386 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1387
1388 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1389 {
1390 if (pGuestMsr->u32Msr == uMsr)
1391 return true;
1392 }
1393 return false;
1394}
1395
1396
1397/**
1398 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1399 *
1400 * @param pVCpu Pointer to the VMCPU.
1401 *
1402 * @remarks No-long-jump zone!!!
1403 */
1404static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1405{
1406 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1407 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1408 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1409 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1410
1411 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1412 {
1413 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1414
1415 /*
1416 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1417 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1418 */
1419 if (pHostMsr->u32Msr == MSR_K6_EFER)
1420 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1421 else
1422 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1423 }
1424
1425 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1426}
1427
1428
1429#if HC_ARCH_BITS == 64
1430/**
1431 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1432 * perform lazy restoration of the host MSRs while leaving VT-x.
1433 *
1434 * @param pVCpu Pointer to the VMCPU.
1435 *
1436 * @remarks No-long-jump zone!!!
1437 */
1438static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1439{
1440 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1441
1442 /*
1443 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1444 */
1445 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1446 {
1447 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1448 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1449 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1450 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1451 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1452 }
1453}
1454
1455
1456/**
1457 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1458 * lazily while leaving VT-x.
1459 *
1460 * @returns true if it does, false otherwise.
1461 * @param pVCpu Pointer to the VMCPU.
1462 * @param uMsr The MSR to check.
1463 */
1464static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1465{
1466 NOREF(pVCpu);
1467 switch (uMsr)
1468 {
1469 case MSR_K8_LSTAR:
1470 case MSR_K6_STAR:
1471 case MSR_K8_SF_MASK:
1472 case MSR_K8_KERNEL_GS_BASE:
1473 return true;
1474 }
1475 return false;
1476}
1477
1478
1479/**
1480 * Saves a set of guest MSRs back into the guest-CPU context.
1481 *
1482 * @param pVCpu Pointer to the VMCPU.
1483 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1484 * out-of-sync. Make sure to update the required fields
1485 * before using them.
1486 *
1487 * @remarks No-long-jump zone!!!
1488 */
1489static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1490{
1491 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1492 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1493
1494 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1495 {
1496 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1497 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1498 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1499 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1500 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1501 }
1502}
1503
1504
1505/**
1506 * Loads a set of guests MSRs to allow read/passthru to the guest.
1507 *
1508 * The name of this function is slightly confusing. This function does NOT
1509 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1510 * common prefix for functions dealing with "lazy restoration" of the shared
1511 * MSRs.
1512 *
1513 * @param pVCpu Pointer to the VMCPU.
1514 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1515 * out-of-sync. Make sure to update the required fields
1516 * before using them.
1517 *
1518 * @remarks No-long-jump zone!!!
1519 */
1520static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1521{
1522 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1523 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1524
1525#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1526 do { \
1527 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1528 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1529 else \
1530 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1531 } while (0)
1532
1533 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1534 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1535 {
1536 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1537 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1538 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1539 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1540 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1541 }
1542 else
1543 {
1544 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1545 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1546 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1547 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1548 }
1549
1550#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1551}
1552
1553
1554/**
1555 * Performs lazy restoration of the set of host MSRs if they were previously
1556 * loaded with guest MSR values.
1557 *
1558 * @param pVCpu Pointer to the VMCPU.
1559 *
1560 * @remarks No-long-jump zone!!!
1561 * @remarks The guest MSRs should have been saved back into the guest-CPU
1562 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1563 */
1564static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1565{
1566 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1567 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1568
1569 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1570 {
1571 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1572 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1573 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1574 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1575 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1576 }
1577 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1578}
1579#endif /* HC_ARCH_BITS == 64 */
1580
1581
1582/**
1583 * Verifies that our cached values of the VMCS controls are all
1584 * consistent with what's actually present in the VMCS.
1585 *
1586 * @returns VBox status code.
1587 * @param pVCpu Pointer to the VMCPU.
1588 */
1589static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1590{
1591 uint32_t u32Val;
1592 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1593 AssertRCReturn(rc, rc);
1594 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1595 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1596
1597 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1598 AssertRCReturn(rc, rc);
1599 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1600 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1601
1602 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1603 AssertRCReturn(rc, rc);
1604 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1605 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1606
1607 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1608 AssertRCReturn(rc, rc);
1609 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1610 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1611
1612 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1613 {
1614 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1615 AssertRCReturn(rc, rc);
1616 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1617 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1618 }
1619
1620 return VINF_SUCCESS;
1621}
1622
1623
1624#ifdef VBOX_STRICT
1625/**
1626 * Verifies that our cached host EFER value has not changed
1627 * since we cached it.
1628 *
1629 * @param pVCpu Pointer to the VMCPU.
1630 */
1631static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1632{
1633 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1634
1635 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1636 {
1637 uint64_t u64Val;
1638 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1639 AssertRC(rc);
1640
1641 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1642 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1643 }
1644}
1645
1646
1647/**
1648 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1649 * VMCS are correct.
1650 *
1651 * @param pVCpu Pointer to the VMCPU.
1652 */
1653static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1654{
1655 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1656
1657 /* Verify MSR counts in the VMCS are what we think it should be. */
1658 uint32_t cMsrs;
1659 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1660 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1661
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1663 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1664
1665 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1666 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1667
1668 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1669 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1670 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1671 {
1672 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1673 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1674 pGuestMsr->u32Msr, cMsrs));
1675
1676 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1677 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1678 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1679
1680 /* Verify that the permissions are as expected in the MSR bitmap. */
1681 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1682 {
1683 VMXMSREXITREAD enmRead;
1684 VMXMSREXITWRITE enmWrite;
1685 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1686 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1687 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1688 {
1689 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1690 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1691 }
1692 else
1693 {
1694 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1695 pGuestMsr->u32Msr, cMsrs));
1696 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1697 pGuestMsr->u32Msr, cMsrs));
1698 }
1699 }
1700 }
1701}
1702#endif /* VBOX_STRICT */
1703
1704
1705/**
1706 * Flushes the TLB using EPT.
1707 *
1708 * @returns VBox status code.
1709 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1710 * enmFlush).
1711 * @param enmFlush Type of flush.
1712 *
1713 * @remarks Caller is responsible for making sure this function is called only
1714 * when NestedPaging is supported and providing @a enmFlush that is
1715 * supported by the CPU.
1716 * @remarks Can be called with interrupts disabled.
1717 */
1718static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1719{
1720 uint64_t au64Descriptor[2];
1721 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1722 au64Descriptor[0] = 0;
1723 else
1724 {
1725 Assert(pVCpu);
1726 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1727 }
1728 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1729
1730 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1731 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1732 rc));
1733 if ( RT_SUCCESS(rc)
1734 && pVCpu)
1735 {
1736 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1737 }
1738}
1739
1740
1741/**
1742 * Flushes the TLB using VPID.
1743 *
1744 * @returns VBox status code.
1745 * @param pVM Pointer to the VM.
1746 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1747 * enmFlush).
1748 * @param enmFlush Type of flush.
1749 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1750 * on @a enmFlush).
1751 *
1752 * @remarks Can be called with interrupts disabled.
1753 */
1754static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1755{
1756 NOREF(pVM);
1757 AssertPtr(pVM);
1758 Assert(pVM->hm.s.vmx.fVpid);
1759
1760 uint64_t au64Descriptor[2];
1761 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1762 {
1763 au64Descriptor[0] = 0;
1764 au64Descriptor[1] = 0;
1765 }
1766 else
1767 {
1768 AssertPtr(pVCpu);
1769 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1770 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1771 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1772 au64Descriptor[1] = GCPtr;
1773 }
1774
1775 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1776 AssertMsg(rc == VINF_SUCCESS,
1777 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1778 if ( RT_SUCCESS(rc)
1779 && pVCpu)
1780 {
1781 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1782 }
1783}
1784
1785
1786/**
1787 * Invalidates a guest page by guest virtual address. Only relevant for
1788 * EPT/VPID, otherwise there is nothing really to invalidate.
1789 *
1790 * @returns VBox status code.
1791 * @param pVM Pointer to the VM.
1792 * @param pVCpu Pointer to the VMCPU.
1793 * @param GCVirt Guest virtual address of the page to invalidate.
1794 */
1795VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1796{
1797 AssertPtr(pVM);
1798 AssertPtr(pVCpu);
1799 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1800
1801 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1802 if (!fFlushPending)
1803 {
1804 /*
1805 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1806 * See @bugref{6043} and @bugref{6177}.
1807 *
1808 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1809 * function maybe called in a loop with individual addresses.
1810 */
1811 if (pVM->hm.s.vmx.fVpid)
1812 {
1813 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1814 {
1815 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1816 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1817 }
1818 else
1819 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1820 }
1821 else if (pVM->hm.s.fNestedPaging)
1822 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1823 }
1824
1825 return VINF_SUCCESS;
1826}
1827
1828
1829/**
1830 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1831 * otherwise there is nothing really to invalidate.
1832 *
1833 * @returns VBox status code.
1834 * @param pVM Pointer to the VM.
1835 * @param pVCpu Pointer to the VMCPU.
1836 * @param GCPhys Guest physical address of the page to invalidate.
1837 */
1838VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1839{
1840 NOREF(pVM); NOREF(GCPhys);
1841 LogFlowFunc(("%RGp\n", GCPhys));
1842
1843 /*
1844 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1845 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1846 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1847 */
1848 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1849 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1850 return VINF_SUCCESS;
1851}
1852
1853
1854/**
1855 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1856 * case where neither EPT nor VPID is supported by the CPU.
1857 *
1858 * @param pVM Pointer to the VM.
1859 * @param pVCpu Pointer to the VMCPU.
1860 * @param pCpu Pointer to the global HM struct.
1861 *
1862 * @remarks Called with interrupts disabled.
1863 */
1864static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1865{
1866 AssertPtr(pVCpu);
1867 AssertPtr(pCpu);
1868 NOREF(pVM);
1869
1870 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1871
1872 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1873#if 0
1874 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1875 pVCpu->hm.s.TlbShootdown.cPages = 0;
1876#endif
1877
1878 Assert(pCpu->idCpu != NIL_RTCPUID);
1879 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1880 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1881 pVCpu->hm.s.fForceTLBFlush = false;
1882 return;
1883}
1884
1885
1886/**
1887 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1888 *
1889 * @param pVM Pointer to the VM.
1890 * @param pVCpu Pointer to the VMCPU.
1891 * @param pCpu Pointer to the global HM CPU struct.
1892 * @remarks All references to "ASID" in this function pertains to "VPID" in
1893 * Intel's nomenclature. The reason is, to avoid confusion in compare
1894 * statements since the host-CPU copies are named "ASID".
1895 *
1896 * @remarks Called with interrupts disabled.
1897 */
1898static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1899{
1900#ifdef VBOX_WITH_STATISTICS
1901 bool fTlbFlushed = false;
1902# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1903# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1904 if (!fTlbFlushed) \
1905 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1906 } while (0)
1907#else
1908# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1909# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1910#endif
1911
1912 AssertPtr(pVM);
1913 AssertPtr(pCpu);
1914 AssertPtr(pVCpu);
1915 Assert(pCpu->idCpu != NIL_RTCPUID);
1916
1917 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1918 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1919 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1920
1921 /*
1922 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1923 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1924 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1925 */
1926 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1927 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1928 {
1929 ++pCpu->uCurrentAsid;
1930 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1931 {
1932 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1933 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1934 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1935 }
1936
1937 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1938 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1939 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1940
1941 /*
1942 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1943 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1944 */
1945 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1946 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1947 HMVMX_SET_TAGGED_TLB_FLUSHED();
1948 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1949 }
1950
1951 /* Check for explicit TLB shootdowns. */
1952 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1953 {
1954 /*
1955 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1956 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1957 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1958 * but not guest-physical mappings.
1959 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1960 */
1961 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1962 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1963 HMVMX_SET_TAGGED_TLB_FLUSHED();
1964 }
1965
1966 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1967 * where it is commented out. Support individual entry flushing
1968 * someday. */
1969#if 0
1970 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1971 {
1972 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1973
1974 /*
1975 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1976 * as supported by the CPU.
1977 */
1978 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1979 {
1980 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1981 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1982 }
1983 else
1984 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1985
1986 HMVMX_SET_TAGGED_TLB_FLUSHED();
1987 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1988 pVCpu->hm.s.TlbShootdown.cPages = 0;
1989 }
1990#endif
1991
1992 pVCpu->hm.s.fForceTLBFlush = false;
1993
1994 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1995
1996 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1997 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1998 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1999 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2000 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2001 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2002 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2003 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2004 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2005
2006 /* Update VMCS with the VPID. */
2007 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2008 AssertRC(rc);
2009
2010#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2011}
2012
2013
2014/**
2015 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2016 *
2017 * @returns VBox status code.
2018 * @param pVM Pointer to the VM.
2019 * @param pVCpu Pointer to the VMCPU.
2020 * @param pCpu Pointer to the global HM CPU struct.
2021 *
2022 * @remarks Called with interrupts disabled.
2023 */
2024static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2025{
2026 AssertPtr(pVM);
2027 AssertPtr(pVCpu);
2028 AssertPtr(pCpu);
2029 Assert(pCpu->idCpu != NIL_RTCPUID);
2030 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2031 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2032
2033 /*
2034 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2035 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2036 */
2037 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2038 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2039 {
2040 pVCpu->hm.s.fForceTLBFlush = true;
2041 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2042 }
2043
2044 /* Check for explicit TLB shootdown flushes. */
2045 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2046 {
2047 pVCpu->hm.s.fForceTLBFlush = true;
2048 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2049 }
2050
2051 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2052 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2053
2054 if (pVCpu->hm.s.fForceTLBFlush)
2055 {
2056 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2057 pVCpu->hm.s.fForceTLBFlush = false;
2058 }
2059 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2060 * where it is commented out. Support individual entry flushing
2061 * someday. */
2062#if 0
2063 else
2064 {
2065 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2066 {
2067 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2068 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2069 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2070 }
2071 else
2072 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2073
2074 pVCpu->hm.s.TlbShootdown.cPages = 0;
2075 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2076 }
2077#endif
2078}
2079
2080
2081/**
2082 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2083 *
2084 * @returns VBox status code.
2085 * @param pVM Pointer to the VM.
2086 * @param pVCpu Pointer to the VMCPU.
2087 * @param pCpu Pointer to the global HM CPU struct.
2088 *
2089 * @remarks Called with interrupts disabled.
2090 */
2091static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2092{
2093 AssertPtr(pVM);
2094 AssertPtr(pVCpu);
2095 AssertPtr(pCpu);
2096 Assert(pCpu->idCpu != NIL_RTCPUID);
2097 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2098 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2099
2100 /*
2101 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2102 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2103 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2104 */
2105 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2106 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2107 {
2108 pVCpu->hm.s.fForceTLBFlush = true;
2109 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2110 }
2111
2112 /* Check for explicit TLB shootdown flushes. */
2113 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2114 {
2115 /*
2116 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2117 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2118 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2119 */
2120 pVCpu->hm.s.fForceTLBFlush = true;
2121 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2122 }
2123
2124 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2125 if (pVCpu->hm.s.fForceTLBFlush)
2126 {
2127 ++pCpu->uCurrentAsid;
2128 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2129 {
2130 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2131 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2132 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2133 }
2134
2135 pVCpu->hm.s.fForceTLBFlush = false;
2136 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2137 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2138 if (pCpu->fFlushAsidBeforeUse)
2139 {
2140 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2142 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2143 {
2144 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2145 pCpu->fFlushAsidBeforeUse = false;
2146 }
2147 else
2148 {
2149 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2150 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2151 }
2152 }
2153 }
2154 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2155 * where it is commented out. Support individual entry flushing
2156 * someday. */
2157#if 0
2158 else
2159 {
2160 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2161 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2162 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2163 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2164
2165 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2166 {
2167 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2168 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2169 {
2170 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2171 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2172 }
2173 else
2174 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2175
2176 pVCpu->hm.s.TlbShootdown.cPages = 0;
2177 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2178 }
2179 else
2180 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2181 }
2182#endif
2183
2184 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2185 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2186 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2187 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2188 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2189 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2190 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2191
2192 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2193 AssertRC(rc);
2194}
2195
2196
2197/**
2198 * Flushes the guest TLB entry based on CPU capabilities.
2199 *
2200 * @param pVCpu Pointer to the VMCPU.
2201 * @param pCpu Pointer to the global HM CPU struct.
2202 */
2203DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2204{
2205#ifdef HMVMX_ALWAYS_FLUSH_TLB
2206 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2207#endif
2208 PVM pVM = pVCpu->CTX_SUFF(pVM);
2209 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2210 {
2211 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2212 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2213 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2214 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2215 default:
2216 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2217 break;
2218 }
2219
2220 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2221 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2222
2223 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2224}
2225
2226
2227/**
2228 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2229 * TLB entries from the host TLB before VM-entry.
2230 *
2231 * @returns VBox status code.
2232 * @param pVM Pointer to the VM.
2233 */
2234static int hmR0VmxSetupTaggedTlb(PVM pVM)
2235{
2236 /*
2237 * Determine optimal flush type for Nested Paging.
2238 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2239 * guest execution (see hmR3InitFinalizeR0()).
2240 */
2241 if (pVM->hm.s.fNestedPaging)
2242 {
2243 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2244 {
2245 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2246 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2247 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2248 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2249 else
2250 {
2251 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2252 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2253 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2254 }
2255
2256 /* Make sure the write-back cacheable memory type for EPT is supported. */
2257 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2258 {
2259 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2260 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2261 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2262 }
2263 }
2264 else
2265 {
2266 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2267 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2268 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2269 }
2270 }
2271
2272 /*
2273 * Determine optimal flush type for VPID.
2274 */
2275 if (pVM->hm.s.vmx.fVpid)
2276 {
2277 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2278 {
2279 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2280 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2281 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2282 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2283 else
2284 {
2285 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2286 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2287 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2288 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2289 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2290 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2291 pVM->hm.s.vmx.fVpid = false;
2292 }
2293 }
2294 else
2295 {
2296 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2297 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2298 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2299 pVM->hm.s.vmx.fVpid = false;
2300 }
2301 }
2302
2303 /*
2304 * Setup the handler for flushing tagged-TLBs.
2305 */
2306 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2307 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2308 else if (pVM->hm.s.fNestedPaging)
2309 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2310 else if (pVM->hm.s.vmx.fVpid)
2311 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2312 else
2313 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2314 return VINF_SUCCESS;
2315}
2316
2317
2318/**
2319 * Sets up pin-based VM-execution controls in the VMCS.
2320 *
2321 * @returns VBox status code.
2322 * @param pVM Pointer to the VM.
2323 * @param pVCpu Pointer to the VMCPU.
2324 */
2325static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2326{
2327 AssertPtr(pVM);
2328 AssertPtr(pVCpu);
2329
2330 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2331 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2332
2333 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2334 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2335
2336 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2337 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2338
2339 /* Enable the VMX preemption timer. */
2340 if (pVM->hm.s.vmx.fUsePreemptTimer)
2341 {
2342 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2343 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2344 }
2345
2346 if ((val & zap) != val)
2347 {
2348 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2349 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2350 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2351 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2352 }
2353
2354 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2355 AssertRCReturn(rc, rc);
2356
2357 pVCpu->hm.s.vmx.u32PinCtls = val;
2358 return rc;
2359}
2360
2361
2362/**
2363 * Sets up processor-based VM-execution controls in the VMCS.
2364 *
2365 * @returns VBox status code.
2366 * @param pVM Pointer to the VM.
2367 * @param pVMCPU Pointer to the VMCPU.
2368 */
2369static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2370{
2371 AssertPtr(pVM);
2372 AssertPtr(pVCpu);
2373
2374 int rc = VERR_INTERNAL_ERROR_5;
2375 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2376 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2377
2378 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2379 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2380 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2381 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2382 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2383 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2384 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2385
2386 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2387 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2388 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2389 {
2390 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2391 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2392 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2393 }
2394
2395 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2396 if (!pVM->hm.s.fNestedPaging)
2397 {
2398 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2399 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2400 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2401 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2402 }
2403
2404 /* Use TPR shadowing if supported by the CPU. */
2405 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2406 {
2407 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2408 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2409 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2410 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2411 AssertRCReturn(rc, rc);
2412
2413 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2414 /* CR8 writes cause a VM-exit based on TPR threshold. */
2415 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2416 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2417 }
2418 else
2419 {
2420 /*
2421 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2422 * Set this control only for 64-bit guests.
2423 */
2424 if (pVM->hm.s.fAllow64BitGuests)
2425 {
2426 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2427 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2428 }
2429 }
2430
2431 /* Use MSR-bitmaps if supported by the CPU. */
2432 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2433 {
2434 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2435
2436 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2437 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2438 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2439 AssertRCReturn(rc, rc);
2440
2441 /*
2442 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2443 * automatically using dedicated fields in the VMCS.
2444 */
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2449 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2450
2451#if HC_ARCH_BITS == 64
2452 /*
2453 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2454 */
2455 if (pVM->hm.s.fAllow64BitGuests)
2456 {
2457 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2458 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2459 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2460 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2461 }
2462#endif
2463 }
2464
2465 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2466 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2467 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2468
2469 if ((val & zap) != val)
2470 {
2471 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2472 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2473 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2474 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2475 }
2476
2477 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2478 AssertRCReturn(rc, rc);
2479
2480 pVCpu->hm.s.vmx.u32ProcCtls = val;
2481
2482 /*
2483 * Secondary processor-based VM-execution controls.
2484 */
2485 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2486 {
2487 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2488 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2489
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2492
2493 if (pVM->hm.s.fNestedPaging)
2494 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2495 else
2496 {
2497 /*
2498 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2499 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2500 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2501 */
2502 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2503 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2504 }
2505
2506 if (pVM->hm.s.vmx.fVpid)
2507 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2508
2509 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2510 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2511
2512 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2513 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2514 * done dynamically. */
2515 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2516 {
2517 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2518 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2519 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2520 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2521 AssertRCReturn(rc, rc);
2522 }
2523
2524 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2525 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2526
2527 if ((val & zap) != val)
2528 {
2529 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2530 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2531 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2532 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2533 }
2534
2535 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2536 AssertRCReturn(rc, rc);
2537
2538 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2539 }
2540 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2541 {
2542 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2543 "available\n"));
2544 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2545 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2546 }
2547
2548 return VINF_SUCCESS;
2549}
2550
2551
2552/**
2553 * Sets up miscellaneous (everything other than Pin & Processor-based
2554 * VM-execution) control fields in the VMCS.
2555 *
2556 * @returns VBox status code.
2557 * @param pVM Pointer to the VM.
2558 * @param pVCpu Pointer to the VMCPU.
2559 */
2560static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2561{
2562 NOREF(pVM);
2563 AssertPtr(pVM);
2564 AssertPtr(pVCpu);
2565
2566 int rc = VERR_GENERAL_FAILURE;
2567
2568 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2569#if 0
2570 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2571 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2572 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2573
2574 /*
2575 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2576 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2577 * We thus use the exception bitmap to control it rather than use both.
2578 */
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2580 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2581
2582 /** @todo Explore possibility of using IO-bitmaps. */
2583 /* All IO & IOIO instructions cause VM-exits. */
2584 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2585 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2586
2587 /* Initialize the MSR-bitmap area. */
2588 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2590 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2591#endif
2592
2593 /* Setup MSR auto-load/store area. */
2594 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2595 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2596 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2597 AssertRCReturn(rc, rc);
2598 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2599 AssertRCReturn(rc, rc);
2600
2601 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2602 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2603 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2604 AssertRCReturn(rc, rc);
2605
2606 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2607 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2608 AssertRCReturn(rc, rc);
2609
2610 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2611#if 0
2612 /* Setup debug controls */
2613 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2614 AssertRCReturn(rc, rc);
2615 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2616 AssertRCReturn(rc, rc);
2617#endif
2618
2619 return rc;
2620}
2621
2622
2623/**
2624 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2625 *
2626 * @returns VBox status code.
2627 * @param pVM Pointer to the VM.
2628 * @param pVCpu Pointer to the VMCPU.
2629 */
2630static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2631{
2632 AssertPtr(pVM);
2633 AssertPtr(pVCpu);
2634
2635 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2636
2637 uint32_t u32XcptBitmap = 0;
2638
2639 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2640 if (!pVM->hm.s.fNestedPaging)
2641 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2642
2643 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2644 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2645 AssertRCReturn(rc, rc);
2646 return rc;
2647}
2648
2649
2650/**
2651 * Sets up the initial guest-state mask. The guest-state mask is consulted
2652 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2653 * for the nested virtualization case (as it would cause a VM-exit).
2654 *
2655 * @param pVCpu Pointer to the VMCPU.
2656 */
2657static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2658{
2659 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2660 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2661 return VINF_SUCCESS;
2662}
2663
2664
2665/**
2666 * Does per-VM VT-x initialization.
2667 *
2668 * @returns VBox status code.
2669 * @param pVM Pointer to the VM.
2670 */
2671VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2672{
2673 LogFlowFunc(("pVM=%p\n", pVM));
2674
2675 int rc = hmR0VmxStructsAlloc(pVM);
2676 if (RT_FAILURE(rc))
2677 {
2678 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2679 return rc;
2680 }
2681
2682 return VINF_SUCCESS;
2683}
2684
2685
2686/**
2687 * Does per-VM VT-x termination.
2688 *
2689 * @returns VBox status code.
2690 * @param pVM Pointer to the VM.
2691 */
2692VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2693{
2694 LogFlowFunc(("pVM=%p\n", pVM));
2695
2696#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2697 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2698 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2699#endif
2700 hmR0VmxStructsFree(pVM);
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Sets up the VM for execution under VT-x.
2707 * This function is only called once per-VM during initialization.
2708 *
2709 * @returns VBox status code.
2710 * @param pVM Pointer to the VM.
2711 */
2712VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2713{
2714 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2715 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2716
2717 LogFlowFunc(("pVM=%p\n", pVM));
2718
2719 /*
2720 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2721 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2722 */
2723 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2724 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2725 || !pVM->hm.s.vmx.pRealModeTSS))
2726 {
2727 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2728 return VERR_INTERNAL_ERROR;
2729 }
2730
2731#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2732 /*
2733 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2734 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2735 */
2736 if ( pVM->hm.s.fAllow64BitGuests
2737 && !HMVMX_IS_64BIT_HOST_MODE())
2738 {
2739 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2740 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2741 }
2742#endif
2743
2744 /* Initialize these always, see hmR3InitFinalizeR0().*/
2745 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2746 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2747
2748 /* Setup the tagged-TLB flush handlers. */
2749 int rc = hmR0VmxSetupTaggedTlb(pVM);
2750 if (RT_FAILURE(rc))
2751 {
2752 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2753 return rc;
2754 }
2755
2756 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2757 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2758#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2759 if ( HMVMX_IS_64BIT_HOST_MODE()
2760 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2761 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2762 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2763 {
2764 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2765 }
2766#endif
2767
2768 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2769 {
2770 PVMCPU pVCpu = &pVM->aCpus[i];
2771 AssertPtr(pVCpu);
2772 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2773
2774 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2775 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2776
2777 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2778 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2779 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2780
2781 /* Set revision dword at the beginning of the VMCS structure. */
2782 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2783
2784 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2785 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2787 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2788
2789 /* Load this VMCS as the current VMCS. */
2790 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2791 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2792 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2793
2794 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2796 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2797
2798 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2800 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2801
2802 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2804 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2805
2806 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2815 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2816 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2817 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2818#endif
2819
2820 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2821 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2822 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2823 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2824
2825 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2826
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2828 }
2829
2830 return VINF_SUCCESS;
2831}
2832
2833
2834/**
2835 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2836 * the VMCS.
2837 *
2838 * @returns VBox status code.
2839 * @param pVM Pointer to the VM.
2840 * @param pVCpu Pointer to the VMCPU.
2841 */
2842DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2843{
2844 NOREF(pVM); NOREF(pVCpu);
2845
2846 RTCCUINTREG uReg = ASMGetCR0();
2847 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2848 AssertRCReturn(rc, rc);
2849
2850#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2851 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2852 if (HMVMX_IS_64BIT_HOST_MODE())
2853 {
2854 uint64_t uRegCR3 = HMR0Get64bitCR3();
2855 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2856 }
2857 else
2858#endif
2859 {
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 }
2863 AssertRCReturn(rc, rc);
2864
2865 uReg = ASMGetCR4();
2866 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2867 AssertRCReturn(rc, rc);
2868 return rc;
2869}
2870
2871
2872#if HC_ARCH_BITS == 64
2873/**
2874 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2875 * requirements. See hmR0VmxSaveHostSegmentRegs().
2876 */
2877# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2878 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2879 { \
2880 bool fValidSelector = true; \
2881 if ((selValue) & X86_SEL_LDT) \
2882 { \
2883 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2884 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2885 } \
2886 if (fValidSelector) \
2887 { \
2888 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2889 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2890 } \
2891 (selValue) = 0; \
2892 }
2893#endif
2894
2895
2896/**
2897 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2898 * the host-state area in the VMCS.
2899 *
2900 * @returns VBox status code.
2901 * @param pVM Pointer to the VM.
2902 * @param pVCpu Pointer to the VMCPU.
2903 */
2904DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2905{
2906 int rc = VERR_INTERNAL_ERROR_5;
2907
2908#if HC_ARCH_BITS == 64
2909 /*
2910 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2911 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2912 */
2913 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2914 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2915#endif
2916
2917 /*
2918 * Host DS, ES, FS and GS segment registers.
2919 */
2920#if HC_ARCH_BITS == 64
2921 RTSEL uSelDS = ASMGetDS();
2922 RTSEL uSelES = ASMGetES();
2923 RTSEL uSelFS = ASMGetFS();
2924 RTSEL uSelGS = ASMGetGS();
2925#else
2926 RTSEL uSelDS = 0;
2927 RTSEL uSelES = 0;
2928 RTSEL uSelFS = 0;
2929 RTSEL uSelGS = 0;
2930#endif
2931
2932 /* Recalculate which host-state bits need to be manually restored. */
2933 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2934
2935 /*
2936 * Host CS and SS segment registers.
2937 */
2938#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2939 RTSEL uSelCS;
2940 RTSEL uSelSS;
2941 if (HMVMX_IS_64BIT_HOST_MODE())
2942 {
2943 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2944 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2945 }
2946 else
2947 {
2948 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2949 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2950 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2951 }
2952#else
2953 RTSEL uSelCS = ASMGetCS();
2954 RTSEL uSelSS = ASMGetSS();
2955#endif
2956
2957 /*
2958 * Host TR segment register.
2959 */
2960 RTSEL uSelTR = ASMGetTR();
2961
2962#if HC_ARCH_BITS == 64
2963 /*
2964 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2965 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2966 */
2967 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2968 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2969 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2970 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2971# undef VMXLOCAL_ADJUST_HOST_SEG
2972#endif
2973
2974 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2975 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2976 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2977 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2978 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2979 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2980 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2981 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2982 Assert(uSelCS);
2983 Assert(uSelTR);
2984
2985 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2986#if 0
2987 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2988 Assert(uSelSS != 0);
2989#endif
2990
2991 /* Write these host selector fields into the host-state area in the VMCS. */
2992 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2994#if HC_ARCH_BITS == 64
2995 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2999#endif
3000 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3001
3002 /*
3003 * Host GDTR and IDTR.
3004 */
3005 RTGDTR Gdtr;
3006 RT_ZERO(Gdtr);
3007#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3008 if (HMVMX_IS_64BIT_HOST_MODE())
3009 {
3010 X86XDTR64 Gdtr64;
3011 X86XDTR64 Idtr64;
3012 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3013 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3014 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3015
3016 Gdtr.cbGdt = Gdtr64.cb;
3017 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3018 }
3019 else
3020#endif
3021 {
3022 RTIDTR Idtr;
3023 ASMGetGDTR(&Gdtr);
3024 ASMGetIDTR(&Idtr);
3025 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3026 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3027
3028#if HC_ARCH_BITS == 64
3029 /*
3030 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3031 * maximum limit (0xffff) on every VM-exit.
3032 */
3033 if (Gdtr.cbGdt != 0xffff)
3034 {
3035 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3036 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3037 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3038 }
3039
3040 /*
3041 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3042 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3043 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3044 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3045 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3046 * hosts where we are pretty sure it won't cause trouble.
3047 */
3048# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3049 if (Idtr.cbIdt < 0x0fff)
3050# else
3051 if (Idtr.cbIdt != 0xffff)
3052# endif
3053 {
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3055 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3056 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3057 }
3058#endif
3059 }
3060
3061 /*
3062 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3063 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3064 */
3065 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3066 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3067 VERR_VMX_INVALID_HOST_STATE);
3068
3069 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3070#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3071 if (HMVMX_IS_64BIT_HOST_MODE())
3072 {
3073 /* We need the 64-bit TR base for hybrid darwin. */
3074 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3075 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3076 }
3077 else
3078#endif
3079 {
3080 uintptr_t uTRBase;
3081#if HC_ARCH_BITS == 64
3082 uTRBase = X86DESC64_BASE(pDesc);
3083
3084 /*
3085 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3086 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3087 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3088 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3089 *
3090 * [1] See Intel spec. 3.5 "System Descriptor Types".
3091 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3092 */
3093 Assert(pDesc->System.u4Type == 11);
3094 if ( pDesc->System.u16LimitLow != 0x67
3095 || pDesc->System.u4LimitHigh)
3096 {
3097 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3098 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3099 if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3100 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3101 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3102
3103 /* Store the GDTR here as we need it while restoring TR. */
3104 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3105 }
3106#else
3107 uTRBase = X86DESC_BASE(pDesc);
3108#endif
3109 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3110 }
3111 AssertRCReturn(rc, rc);
3112
3113 /*
3114 * Host FS base and GS base.
3115 */
3116#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3117 if (HMVMX_IS_64BIT_HOST_MODE())
3118 {
3119 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3120 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3121 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3122 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3123
3124# if HC_ARCH_BITS == 64
3125 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3126 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3127 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3128 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3129 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3130# endif
3131 }
3132#endif
3133 return rc;
3134}
3135
3136
3137/**
3138 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3139 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3140 * the host after every successful VM-exit.
3141 *
3142 * @returns VBox status code.
3143 * @param pVM Pointer to the VM.
3144 * @param pVCpu Pointer to the VMCPU.
3145 *
3146 * @remarks No-long-jump zone!!!
3147 */
3148DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3149{
3150 NOREF(pVM);
3151
3152 AssertPtr(pVCpu);
3153 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3154
3155 int rc = VINF_SUCCESS;
3156#if HC_ARCH_BITS == 64
3157 if (pVM->hm.s.fAllow64BitGuests)
3158 hmR0VmxLazySaveHostMsrs(pVCpu);
3159#endif
3160
3161 /*
3162 * Host Sysenter MSRs.
3163 */
3164 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3165 AssertRCReturn(rc, rc);
3166#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3167 if (HMVMX_IS_64BIT_HOST_MODE())
3168 {
3169 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3170 AssertRCReturn(rc, rc);
3171 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3172 }
3173 else
3174 {
3175 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3176 AssertRCReturn(rc, rc);
3177 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3178 }
3179#elif HC_ARCH_BITS == 32
3180 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3181 AssertRCReturn(rc, rc);
3182 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3183#else
3184 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3185 AssertRCReturn(rc, rc);
3186 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3187#endif
3188 AssertRCReturn(rc, rc);
3189
3190 /*
3191 * Host EFER MSR.
3192 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3193 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3194 */
3195 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3196 {
3197 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3198 AssertRCReturn(rc, rc);
3199 }
3200
3201 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3202 * hmR0VmxLoadGuestExitCtls() !! */
3203
3204 return rc;
3205}
3206
3207
3208/**
3209 * Figures out if we need to swap the EFER MSR which is
3210 * particularly expensive.
3211 *
3212 * We check all relevant bits. For now, that's everything
3213 * besides LMA/LME, as these two bits are handled by VM-entry,
3214 * see hmR0VmxLoadGuestExitCtls() and
3215 * hmR0VMxLoadGuestEntryCtls().
3216 *
3217 * @returns true if we need to load guest EFER, false otherwise.
3218 * @param pVCpu Pointer to the VMCPU.
3219 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3220 * out-of-sync. Make sure to update the required fields
3221 * before using them.
3222 *
3223 * @remarks Requires EFER, CR4.
3224 * @remarks No-long-jump zone!!!
3225 */
3226static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3227{
3228#ifdef HMVMX_ALWAYS_SWAP_EFER
3229 return true;
3230#endif
3231
3232#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3233 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3234 if (CPUMIsGuestInLongMode(pVCpu))
3235 return false;
3236#endif
3237
3238 PVM pVM = pVCpu->CTX_SUFF(pVM);
3239 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3240 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3241
3242 /*
3243 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3244 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3245 */
3246 if ( CPUMIsGuestInLongMode(pVCpu)
3247 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3248 {
3249 return true;
3250 }
3251
3252 /*
3253 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3254 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3255 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3256 */
3257 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3258 && (pMixedCtx->cr0 & X86_CR0_PG)
3259 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3260 {
3261 /* Assert that host is PAE capable. */
3262 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3263 return true;
3264 }
3265
3266 /** @todo Check the latest Intel spec. for any other bits,
3267 * like SMEP/SMAP? */
3268 return false;
3269}
3270
3271
3272/**
3273 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3274 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3275 * controls".
3276 *
3277 * @returns VBox status code.
3278 * @param pVCpu Pointer to the VMCPU.
3279 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3280 * out-of-sync. Make sure to update the required fields
3281 * before using them.
3282 *
3283 * @remarks Requires EFER.
3284 * @remarks No-long-jump zone!!!
3285 */
3286DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3287{
3288 int rc = VINF_SUCCESS;
3289 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3290 {
3291 PVM pVM = pVCpu->CTX_SUFF(pVM);
3292 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3293 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3294
3295 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3296 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3297
3298 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3299 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3300 {
3301 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3302 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3303 }
3304 else
3305 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3306
3307 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3308 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3309 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3310 {
3311 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3312 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3313 }
3314
3315 /*
3316 * The following should -not- be set (since we're not in SMM mode):
3317 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3318 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3319 */
3320
3321 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3322 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3323
3324 if ((val & zap) != val)
3325 {
3326 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3327 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3328 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3329 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3330 }
3331
3332 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3333 AssertRCReturn(rc, rc);
3334
3335 pVCpu->hm.s.vmx.u32EntryCtls = val;
3336 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3337 }
3338 return rc;
3339}
3340
3341
3342/**
3343 * Sets up the VM-exit controls in the VMCS.
3344 *
3345 * @returns VBox status code.
3346 * @param pVM Pointer to the VM.
3347 * @param pVCpu Pointer to the VMCPU.
3348 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3349 * out-of-sync. Make sure to update the required fields
3350 * before using them.
3351 *
3352 * @remarks Requires EFER.
3353 */
3354DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3355{
3356 NOREF(pMixedCtx);
3357
3358 int rc = VINF_SUCCESS;
3359 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3360 {
3361 PVM pVM = pVCpu->CTX_SUFF(pVM);
3362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3364
3365 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3366 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3367
3368 /*
3369 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3370 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3371 */
3372#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3373 if (HMVMX_IS_64BIT_HOST_MODE())
3374 {
3375 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3376 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3377 }
3378 else
3379 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3380#else
3381 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3382 {
3383 /* The switcher returns to long mode, EFER is managed by the switcher. */
3384 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3385 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3386 }
3387 else
3388 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3389#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3390
3391 /* If the newer VMCS fields for managing EFER exists, use it. */
3392 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3393 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3394 {
3395 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3396 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3397 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3398 }
3399
3400 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3401 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3402
3403 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3404 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3405 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3406
3407 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3408 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3409
3410 if ((val & zap) != val)
3411 {
3412 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3413 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3414 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3415 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3416 }
3417
3418 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3419 AssertRCReturn(rc, rc);
3420
3421 pVCpu->hm.s.vmx.u32ExitCtls = val;
3422 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3423 }
3424 return rc;
3425}
3426
3427
3428/**
3429 * Loads the guest APIC and related state.
3430 *
3431 * @returns VBox status code.
3432 * @param pVM Pointer to the VM.
3433 * @param pVCpu Pointer to the VMCPU.
3434 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3435 * out-of-sync. Make sure to update the required fields
3436 * before using them.
3437 */
3438DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3439{
3440 NOREF(pMixedCtx);
3441
3442 int rc = VINF_SUCCESS;
3443 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3444 {
3445 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3446 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3447 {
3448 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3449
3450 bool fPendingIntr = false;
3451 uint8_t u8Tpr = 0;
3452 uint8_t u8PendingIntr = 0;
3453 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3454 AssertRCReturn(rc, rc);
3455
3456 /*
3457 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3458 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3459 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3460 * the interrupt when we VM-exit for other reasons.
3461 */
3462 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3463 uint32_t u32TprThreshold = 0;
3464 if (fPendingIntr)
3465 {
3466 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3467 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3468 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3469 if (u8PendingPriority <= u8TprPriority)
3470 u32TprThreshold = u8PendingPriority;
3471 else
3472 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3473 }
3474 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3475
3476 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3477 AssertRCReturn(rc, rc);
3478 }
3479
3480 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3481 }
3482 return rc;
3483}
3484
3485
3486/**
3487 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3488 *
3489 * @returns Guest's interruptibility-state.
3490 * @param pVCpu Pointer to the VMCPU.
3491 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3492 * out-of-sync. Make sure to update the required fields
3493 * before using them.
3494 *
3495 * @remarks No-long-jump zone!!!
3496 */
3497DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3498{
3499 /*
3500 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3501 */
3502 uint32_t uIntrState = 0;
3503 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3504 {
3505 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3506 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3507 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3508 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3509 {
3510 if (pMixedCtx->eflags.Bits.u1IF)
3511 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3512 else
3513 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3514 }
3515 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3516 }
3517
3518 /*
3519 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3520 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3521 * setting this would block host-NMIs and IRET will not clear the blocking.
3522 *
3523 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3524 */
3525 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3526 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3527 {
3528 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3529 }
3530
3531 return uIntrState;
3532}
3533
3534
3535/**
3536 * Loads the guest's interruptibility-state into the guest-state area in the
3537 * VMCS.
3538 *
3539 * @returns VBox status code.
3540 * @param pVCpu Pointer to the VMCPU.
3541 * @param uIntrState The interruptibility-state to set.
3542 */
3543static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3544{
3545 NOREF(pVCpu);
3546 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3547 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3548 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3549 AssertRCReturn(rc, rc);
3550 return rc;
3551}
3552
3553
3554/**
3555 * Loads the guest's RIP into the guest-state area in the VMCS.
3556 *
3557 * @returns VBox status code.
3558 * @param pVCpu Pointer to the VMCPU.
3559 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3560 * out-of-sync. Make sure to update the required fields
3561 * before using them.
3562 *
3563 * @remarks No-long-jump zone!!!
3564 */
3565static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3566{
3567 int rc = VINF_SUCCESS;
3568 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3569 {
3570 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3571 AssertRCReturn(rc, rc);
3572
3573 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3574 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3575 HMCPU_CF_VALUE(pVCpu)));
3576 }
3577 return rc;
3578}
3579
3580
3581/**
3582 * Loads the guest's RSP into the guest-state area in the VMCS.
3583 *
3584 * @returns VBox status code.
3585 * @param pVCpu Pointer to the VMCPU.
3586 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3587 * out-of-sync. Make sure to update the required fields
3588 * before using them.
3589 *
3590 * @remarks No-long-jump zone!!!
3591 */
3592static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3593{
3594 int rc = VINF_SUCCESS;
3595 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3596 {
3597 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3598 AssertRCReturn(rc, rc);
3599
3600 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3601 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3602 }
3603 return rc;
3604}
3605
3606
3607/**
3608 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3609 *
3610 * @returns VBox status code.
3611 * @param pVCpu Pointer to the VMCPU.
3612 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3613 * out-of-sync. Make sure to update the required fields
3614 * before using them.
3615 *
3616 * @remarks No-long-jump zone!!!
3617 */
3618static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3619{
3620 int rc = VINF_SUCCESS;
3621 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3622 {
3623 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3624 Let us assert it as such and use 32-bit VMWRITE. */
3625 Assert(!(pMixedCtx->rflags.u64 >> 32));
3626 X86EFLAGS Eflags = pMixedCtx->eflags;
3627 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3628 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3629 * These will never be cleared/set, unless some other part of the VMM
3630 * code is buggy - in which case we're better of finding and fixing
3631 * those bugs than hiding them. */
3632 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3633 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3634 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3635 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3636
3637 /*
3638 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3639 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3640 */
3641 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3642 {
3643 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3644 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3645 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3646 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3647 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3648 }
3649
3650 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3651 AssertRCReturn(rc, rc);
3652
3653 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3654 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3655 }
3656 return rc;
3657}
3658
3659
3660/**
3661 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3662 *
3663 * @returns VBox status code.
3664 * @param pVCpu Pointer to the VMCPU.
3665 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3666 * out-of-sync. Make sure to update the required fields
3667 * before using them.
3668 *
3669 * @remarks No-long-jump zone!!!
3670 */
3671DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3672{
3673 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3674 AssertRCReturn(rc, rc);
3675 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3676 AssertRCReturn(rc, rc);
3677 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3678 AssertRCReturn(rc, rc);
3679 return rc;
3680}
3681
3682
3683/**
3684 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3685 * CR0 is partially shared with the host and we have to consider the FPU bits.
3686 *
3687 * @returns VBox status code.
3688 * @param pVM Pointer to the VM.
3689 * @param pVCpu Pointer to the VMCPU.
3690 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3691 * out-of-sync. Make sure to update the required fields
3692 * before using them.
3693 *
3694 * @remarks No-long-jump zone!!!
3695 */
3696static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3697{
3698 /*
3699 * Guest CR0.
3700 * Guest FPU.
3701 */
3702 int rc = VINF_SUCCESS;
3703 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3704 {
3705 Assert(!(pMixedCtx->cr0 >> 32));
3706 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3707 PVM pVM = pVCpu->CTX_SUFF(pVM);
3708
3709 /* The guest's view (read access) of its CR0 is unblemished. */
3710 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3711 AssertRCReturn(rc, rc);
3712 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3713
3714 /* Setup VT-x's view of the guest CR0. */
3715 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3716 if (pVM->hm.s.fNestedPaging)
3717 {
3718 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3719 {
3720 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3721 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3723 }
3724 else
3725 {
3726 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3727 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3728 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729 }
3730
3731 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3732 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3733 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734
3735 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3736 AssertRCReturn(rc, rc);
3737 }
3738 else
3739 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3740
3741 /*
3742 * Guest FPU bits.
3743 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3744 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3745 */
3746 u32GuestCR0 |= X86_CR0_NE;
3747 bool fInterceptNM = false;
3748 if (CPUMIsGuestFPUStateActive(pVCpu))
3749 {
3750 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3751 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3752 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3753 }
3754 else
3755 {
3756 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3757 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3758 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3759 }
3760
3761 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3762 bool fInterceptMF = false;
3763 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3764 fInterceptMF = true;
3765
3766 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3767 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3768 {
3769 Assert(PDMVmmDevHeapIsEnabled(pVM));
3770 Assert(pVM->hm.s.vmx.pRealModeTSS);
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3772 fInterceptNM = true;
3773 fInterceptMF = true;
3774 }
3775
3776 if (fInterceptNM)
3777 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3778 else
3779 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3780
3781 if (fInterceptMF)
3782 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3783 else
3784 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3785
3786 /* Additional intercepts for debugging, define these yourself explicitly. */
3787#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3788 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3789 | RT_BIT(X86_XCPT_BP)
3790 | RT_BIT(X86_XCPT_DB)
3791 | RT_BIT(X86_XCPT_DE)
3792 | RT_BIT(X86_XCPT_NM)
3793 | RT_BIT(X86_XCPT_TS)
3794 | RT_BIT(X86_XCPT_UD)
3795 | RT_BIT(X86_XCPT_NP)
3796 | RT_BIT(X86_XCPT_SS)
3797 | RT_BIT(X86_XCPT_GP)
3798 | RT_BIT(X86_XCPT_PF)
3799 | RT_BIT(X86_XCPT_MF)
3800 ;
3801#elif defined(HMVMX_ALWAYS_TRAP_PF)
3802 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3803#endif
3804
3805 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3806
3807 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3808 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3809 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3810 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3811 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3812 else
3813 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3814
3815 u32GuestCR0 |= uSetCR0;
3816 u32GuestCR0 &= uZapCR0;
3817 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3818
3819 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3820 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3821 AssertRCReturn(rc, rc);
3822 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3823 AssertRCReturn(rc, rc);
3824 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3825 uZapCR0));
3826
3827 /*
3828 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3829 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3830 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3831 */
3832 uint32_t u32CR0Mask = 0;
3833 u32CR0Mask = X86_CR0_PE
3834 | X86_CR0_NE
3835 | X86_CR0_WP
3836 | X86_CR0_PG
3837 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3838 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3839 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3840
3841 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3842 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3843 * and @bugref{6944}. */
3844#if 0
3845 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3846 u32CR0Mask &= ~X86_CR0_PE;
3847#endif
3848 if (pVM->hm.s.fNestedPaging)
3849 u32CR0Mask &= ~X86_CR0_WP;
3850
3851 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3852 if (fInterceptNM)
3853 {
3854 u32CR0Mask |= X86_CR0_TS
3855 | X86_CR0_MP;
3856 }
3857
3858 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3859 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3860 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3861 AssertRCReturn(rc, rc);
3862 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3863
3864 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3865 }
3866 return rc;
3867}
3868
3869
3870/**
3871 * Loads the guest control registers (CR3, CR4) into the guest-state area
3872 * in the VMCS.
3873 *
3874 * @returns VBox status code.
3875 * @param pVM Pointer to the VM.
3876 * @param pVCpu Pointer to the VMCPU.
3877 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3878 * out-of-sync. Make sure to update the required fields
3879 * before using them.
3880 *
3881 * @remarks No-long-jump zone!!!
3882 */
3883static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3884{
3885 int rc = VINF_SUCCESS;
3886 PVM pVM = pVCpu->CTX_SUFF(pVM);
3887
3888 /*
3889 * Guest CR2.
3890 * It's always loaded in the assembler code. Nothing to do here.
3891 */
3892
3893 /*
3894 * Guest CR3.
3895 */
3896 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3897 {
3898 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3899 if (pVM->hm.s.fNestedPaging)
3900 {
3901 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3902
3903 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3904 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3905 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3906 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3907
3908 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3909 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3910 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3911
3912 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3913 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3914 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3915 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3916
3917 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3918 AssertRCReturn(rc, rc);
3919 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3920
3921 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3922 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3923 {
3924 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3925 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3926 {
3927 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3928 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3929 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3930 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3931 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3932 }
3933
3934 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3935 have Unrestricted Execution to handle the guest when it's not using paging. */
3936 GCPhysGuestCR3 = pMixedCtx->cr3;
3937 }
3938 else
3939 {
3940 /*
3941 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3942 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3943 * EPT takes care of translating it to host-physical addresses.
3944 */
3945 RTGCPHYS GCPhys;
3946 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3947 Assert(PDMVmmDevHeapIsEnabled(pVM));
3948
3949 /* We obtain it here every time as the guest could have relocated this PCI region. */
3950 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3951 AssertRCReturn(rc, rc);
3952
3953 GCPhysGuestCR3 = GCPhys;
3954 }
3955
3956 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3957 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3958 }
3959 else
3960 {
3961 /* Non-nested paging case, just use the hypervisor's CR3. */
3962 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3963
3964 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3965 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3966 }
3967 AssertRCReturn(rc, rc);
3968
3969 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3970 }
3971
3972 /*
3973 * Guest CR4.
3974 */
3975 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3976 {
3977 Assert(!(pMixedCtx->cr4 >> 32));
3978 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3979
3980 /* The guest's view of its CR4 is unblemished. */
3981 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3982 AssertRCReturn(rc, rc);
3983 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3984
3985 /* Setup VT-x's view of the guest CR4. */
3986 /*
3987 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3988 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3989 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3990 */
3991 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3992 {
3993 Assert(pVM->hm.s.vmx.pRealModeTSS);
3994 Assert(PDMVmmDevHeapIsEnabled(pVM));
3995 u32GuestCR4 &= ~X86_CR4_VME;
3996 }
3997
3998 if (pVM->hm.s.fNestedPaging)
3999 {
4000 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4001 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4002 {
4003 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4004 u32GuestCR4 |= X86_CR4_PSE;
4005 /* Our identity mapping is a 32-bit page directory. */
4006 u32GuestCR4 &= ~X86_CR4_PAE;
4007 }
4008 /* else use guest CR4.*/
4009 }
4010 else
4011 {
4012 /*
4013 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4014 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4015 */
4016 switch (pVCpu->hm.s.enmShadowMode)
4017 {
4018 case PGMMODE_REAL: /* Real-mode. */
4019 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4020 case PGMMODE_32_BIT: /* 32-bit paging. */
4021 {
4022 u32GuestCR4 &= ~X86_CR4_PAE;
4023 break;
4024 }
4025
4026 case PGMMODE_PAE: /* PAE paging. */
4027 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4028 {
4029 u32GuestCR4 |= X86_CR4_PAE;
4030 break;
4031 }
4032
4033 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4034 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4035#ifdef VBOX_ENABLE_64_BITS_GUESTS
4036 break;
4037#endif
4038 default:
4039 AssertFailed();
4040 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4041 }
4042 }
4043
4044 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4045 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4046 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4047 u32GuestCR4 |= uSetCR4;
4048 u32GuestCR4 &= uZapCR4;
4049
4050 /* Write VT-x's view of the guest CR4 into the VMCS. */
4051 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4052 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4053 AssertRCReturn(rc, rc);
4054
4055 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4056 uint32_t u32CR4Mask = 0;
4057 u32CR4Mask = X86_CR4_VME
4058 | X86_CR4_PAE
4059 | X86_CR4_PGE
4060 | X86_CR4_PSE
4061 | X86_CR4_VMXE;
4062 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4063 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4064 AssertRCReturn(rc, rc);
4065
4066 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4067 }
4068 return rc;
4069}
4070
4071
4072/**
4073 * Loads the guest debug registers into the guest-state area in the VMCS.
4074 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4075 *
4076 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4077 *
4078 * @returns VBox status code.
4079 * @param pVCpu Pointer to the VMCPU.
4080 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4081 * out-of-sync. Make sure to update the required fields
4082 * before using them.
4083 *
4084 * @remarks No-long-jump zone!!!
4085 */
4086static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4087{
4088 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4089 return VINF_SUCCESS;
4090
4091#ifdef VBOX_STRICT
4092 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4093 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4094 {
4095 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4096 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4097 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4098 }
4099#endif
4100
4101 int rc;
4102 PVM pVM = pVCpu->CTX_SUFF(pVM);
4103 bool fInterceptDB = false;
4104 bool fInterceptMovDRx = false;
4105 if ( pVCpu->hm.s.fSingleInstruction
4106 || DBGFIsStepping(pVCpu))
4107 {
4108 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4109 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4110 {
4111 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4112 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4113 AssertRCReturn(rc, rc);
4114 Assert(fInterceptDB == false);
4115 }
4116 else
4117 {
4118 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4119 pVCpu->hm.s.fClearTrapFlag = true;
4120 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4121 fInterceptDB = true;
4122 }
4123 }
4124
4125 if ( fInterceptDB
4126 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4127 {
4128 /*
4129 * Use the combined guest and host DRx values found in the hypervisor
4130 * register set because the debugger has breakpoints active or someone
4131 * is single stepping on the host side without a monitor trap flag.
4132 *
4133 * Note! DBGF expects a clean DR6 state before executing guest code.
4134 */
4135#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4136 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4137 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4138 {
4139 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4140 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4141 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4142 }
4143 else
4144#endif
4145 if (!CPUMIsHyperDebugStateActive(pVCpu))
4146 {
4147 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4148 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4149 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4150 }
4151
4152 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4153 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4154 AssertRCReturn(rc, rc);
4155
4156 pVCpu->hm.s.fUsingHyperDR7 = true;
4157 fInterceptDB = true;
4158 fInterceptMovDRx = true;
4159 }
4160 else
4161 {
4162 /*
4163 * If the guest has enabled debug registers, we need to load them prior to
4164 * executing guest code so they'll trigger at the right time.
4165 */
4166 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4167 {
4168#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4169 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4170 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4171 {
4172 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4173 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4174 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4175 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4176 }
4177 else
4178#endif
4179 if (!CPUMIsGuestDebugStateActive(pVCpu))
4180 {
4181 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4182 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4183 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4184 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4185 }
4186 Assert(!fInterceptDB);
4187 Assert(!fInterceptMovDRx);
4188 }
4189 /*
4190 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4191 * must intercept #DB in order to maintain a correct DR6 guest value.
4192 */
4193#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4194 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4195 && !CPUMIsGuestDebugStateActive(pVCpu))
4196#else
4197 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4198#endif
4199 {
4200 fInterceptMovDRx = true;
4201 fInterceptDB = true;
4202 }
4203
4204 /* Update guest DR7. */
4205 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4206 AssertRCReturn(rc, rc);
4207
4208 pVCpu->hm.s.fUsingHyperDR7 = false;
4209 }
4210
4211 /*
4212 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4213 */
4214 if (fInterceptDB)
4215 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4216 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4217 {
4218#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4219 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4220#endif
4221 }
4222 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4223 AssertRCReturn(rc, rc);
4224
4225 /*
4226 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4227 */
4228 if (fInterceptMovDRx)
4229 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4230 else
4231 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4232 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4233 AssertRCReturn(rc, rc);
4234
4235 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4236 return VINF_SUCCESS;
4237}
4238
4239
4240#ifdef VBOX_STRICT
4241/**
4242 * Strict function to validate segment registers.
4243 *
4244 * @remarks ASSUMES CR0 is up to date.
4245 */
4246static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4247{
4248 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4249 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4250 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4251 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4252 && ( !CPUMIsGuestInRealModeEx(pCtx)
4253 && !CPUMIsGuestInV86ModeEx(pCtx)))
4254 {
4255 /* Protected mode checks */
4256 /* CS */
4257 Assert(pCtx->cs.Attr.n.u1Present);
4258 Assert(!(pCtx->cs.Attr.u & 0xf00));
4259 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4260 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4261 || !(pCtx->cs.Attr.n.u1Granularity));
4262 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4263 || (pCtx->cs.Attr.n.u1Granularity));
4264 /* CS cannot be loaded with NULL in protected mode. */
4265 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4266 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4267 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4268 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4269 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4270 else
4271 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4272 /* SS */
4273 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4274 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4275 if ( !(pCtx->cr0 & X86_CR0_PE)
4276 || pCtx->cs.Attr.n.u4Type == 3)
4277 {
4278 Assert(!pCtx->ss.Attr.n.u2Dpl);
4279 }
4280 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4281 {
4282 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4283 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4284 Assert(pCtx->ss.Attr.n.u1Present);
4285 Assert(!(pCtx->ss.Attr.u & 0xf00));
4286 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4287 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4288 || !(pCtx->ss.Attr.n.u1Granularity));
4289 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4290 || (pCtx->ss.Attr.n.u1Granularity));
4291 }
4292 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4293 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4294 {
4295 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4296 Assert(pCtx->ds.Attr.n.u1Present);
4297 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4298 Assert(!(pCtx->ds.Attr.u & 0xf00));
4299 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4300 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4301 || !(pCtx->ds.Attr.n.u1Granularity));
4302 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4303 || (pCtx->ds.Attr.n.u1Granularity));
4304 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4305 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4306 }
4307 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4308 {
4309 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4310 Assert(pCtx->es.Attr.n.u1Present);
4311 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4312 Assert(!(pCtx->es.Attr.u & 0xf00));
4313 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4314 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4315 || !(pCtx->es.Attr.n.u1Granularity));
4316 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4317 || (pCtx->es.Attr.n.u1Granularity));
4318 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4319 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4320 }
4321 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4322 {
4323 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4324 Assert(pCtx->fs.Attr.n.u1Present);
4325 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4326 Assert(!(pCtx->fs.Attr.u & 0xf00));
4327 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4328 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4329 || !(pCtx->fs.Attr.n.u1Granularity));
4330 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4331 || (pCtx->fs.Attr.n.u1Granularity));
4332 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4333 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4334 }
4335 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4336 {
4337 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4338 Assert(pCtx->gs.Attr.n.u1Present);
4339 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4340 Assert(!(pCtx->gs.Attr.u & 0xf00));
4341 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4342 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4343 || !(pCtx->gs.Attr.n.u1Granularity));
4344 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4345 || (pCtx->gs.Attr.n.u1Granularity));
4346 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4347 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4348 }
4349 /* 64-bit capable CPUs. */
4350# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4351 if (HMVMX_IS_64BIT_HOST_MODE())
4352 {
4353 Assert(!(pCtx->cs.u64Base >> 32));
4354 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4355 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4356 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4357 }
4358# endif
4359 }
4360 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4361 || ( CPUMIsGuestInRealModeEx(pCtx)
4362 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4363 {
4364 /* Real and v86 mode checks. */
4365 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4366 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4367 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4368 {
4369 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4370 }
4371 else
4372 {
4373 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4374 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4375 }
4376
4377 /* CS */
4378 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4379 Assert(pCtx->cs.u32Limit == 0xffff);
4380 Assert(u32CSAttr == 0xf3);
4381 /* SS */
4382 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4383 Assert(pCtx->ss.u32Limit == 0xffff);
4384 Assert(u32SSAttr == 0xf3);
4385 /* DS */
4386 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4387 Assert(pCtx->ds.u32Limit == 0xffff);
4388 Assert(u32DSAttr == 0xf3);
4389 /* ES */
4390 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4391 Assert(pCtx->es.u32Limit == 0xffff);
4392 Assert(u32ESAttr == 0xf3);
4393 /* FS */
4394 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4395 Assert(pCtx->fs.u32Limit == 0xffff);
4396 Assert(u32FSAttr == 0xf3);
4397 /* GS */
4398 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4399 Assert(pCtx->gs.u32Limit == 0xffff);
4400 Assert(u32GSAttr == 0xf3);
4401 /* 64-bit capable CPUs. */
4402# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4403 if (HMVMX_IS_64BIT_HOST_MODE())
4404 {
4405 Assert(!(pCtx->cs.u64Base >> 32));
4406 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4407 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4408 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4409 }
4410# endif
4411 }
4412}
4413#endif /* VBOX_STRICT */
4414
4415
4416/**
4417 * Writes a guest segment register into the guest-state area in the VMCS.
4418 *
4419 * @returns VBox status code.
4420 * @param pVCpu Pointer to the VMCPU.
4421 * @param idxSel Index of the selector in the VMCS.
4422 * @param idxLimit Index of the segment limit in the VMCS.
4423 * @param idxBase Index of the segment base in the VMCS.
4424 * @param idxAccess Index of the access rights of the segment in the VMCS.
4425 * @param pSelReg Pointer to the segment selector.
4426 *
4427 * @remarks No-long-jump zone!!!
4428 */
4429static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4430 uint32_t idxAccess, PCPUMSELREG pSelReg)
4431{
4432 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4433 AssertRCReturn(rc, rc);
4434 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4435 AssertRCReturn(rc, rc);
4436 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4437 AssertRCReturn(rc, rc);
4438
4439 uint32_t u32Access = pSelReg->Attr.u;
4440 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4441 {
4442 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4443 u32Access = 0xf3;
4444 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4445 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4446 }
4447 else
4448 {
4449 /*
4450 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4451 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4452 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4453 * loaded in protected-mode have their attribute as 0.
4454 */
4455 if (!u32Access)
4456 u32Access = X86DESCATTR_UNUSABLE;
4457 }
4458
4459 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4460 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4461 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4462
4463 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4464 AssertRCReturn(rc, rc);
4465 return rc;
4466}
4467
4468
4469/**
4470 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4471 * into the guest-state area in the VMCS.
4472 *
4473 * @returns VBox status code.
4474 * @param pVM Pointer to the VM.
4475 * @param pVCPU Pointer to the VMCPU.
4476 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4477 * out-of-sync. Make sure to update the required fields
4478 * before using them.
4479 *
4480 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4481 * @remarks No-long-jump zone!!!
4482 */
4483static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4484{
4485 int rc = VERR_INTERNAL_ERROR_5;
4486 PVM pVM = pVCpu->CTX_SUFF(pVM);
4487
4488 /*
4489 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4490 */
4491 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4492 {
4493 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4494 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4495 {
4496 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4497 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4498 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4501 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4502 }
4503
4504#ifdef VBOX_WITH_REM
4505 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4506 {
4507 Assert(pVM->hm.s.vmx.pRealModeTSS);
4508 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4509 if ( pVCpu->hm.s.vmx.fWasInRealMode
4510 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4511 {
4512 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4513 in real-mode (e.g. OpenBSD 4.0) */
4514 REMFlushTBs(pVM);
4515 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4516 pVCpu->hm.s.vmx.fWasInRealMode = false;
4517 }
4518 }
4519#endif
4520 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4521 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4522 AssertRCReturn(rc, rc);
4523 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4524 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4525 AssertRCReturn(rc, rc);
4526 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4527 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4528 AssertRCReturn(rc, rc);
4529 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4530 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4531 AssertRCReturn(rc, rc);
4532 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4533 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4534 AssertRCReturn(rc, rc);
4535 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4536 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4537 AssertRCReturn(rc, rc);
4538
4539#ifdef VBOX_STRICT
4540 /* Validate. */
4541 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4542#endif
4543
4544 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4545 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4546 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4547 }
4548
4549 /*
4550 * Guest TR.
4551 */
4552 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4553 {
4554 /*
4555 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4556 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4557 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4558 */
4559 uint16_t u16Sel = 0;
4560 uint32_t u32Limit = 0;
4561 uint64_t u64Base = 0;
4562 uint32_t u32AccessRights = 0;
4563
4564 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4565 {
4566 u16Sel = pMixedCtx->tr.Sel;
4567 u32Limit = pMixedCtx->tr.u32Limit;
4568 u64Base = pMixedCtx->tr.u64Base;
4569 u32AccessRights = pMixedCtx->tr.Attr.u;
4570 }
4571 else
4572 {
4573 Assert(pVM->hm.s.vmx.pRealModeTSS);
4574 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4575
4576 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4577 RTGCPHYS GCPhys;
4578 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4579 AssertRCReturn(rc, rc);
4580
4581 X86DESCATTR DescAttr;
4582 DescAttr.u = 0;
4583 DescAttr.n.u1Present = 1;
4584 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4585
4586 u16Sel = 0;
4587 u32Limit = HM_VTX_TSS_SIZE;
4588 u64Base = GCPhys; /* in real-mode phys = virt. */
4589 u32AccessRights = DescAttr.u;
4590 }
4591
4592 /* Validate. */
4593 Assert(!(u16Sel & RT_BIT(2)));
4594 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4595 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4596 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4597 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4598 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4599 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4600 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4601 Assert( (u32Limit & 0xfff) == 0xfff
4602 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4603 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4604 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4605
4606 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4607 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4608 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4609 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4610
4611 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4612 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4613 }
4614
4615 /*
4616 * Guest GDTR.
4617 */
4618 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4619 {
4620 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4621 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4622
4623 /* Validate. */
4624 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4625
4626 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4627 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4628 }
4629
4630 /*
4631 * Guest LDTR.
4632 */
4633 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4634 {
4635 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4636 uint32_t u32Access = 0;
4637 if (!pMixedCtx->ldtr.Attr.u)
4638 u32Access = X86DESCATTR_UNUSABLE;
4639 else
4640 u32Access = pMixedCtx->ldtr.Attr.u;
4641
4642 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4643 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4644 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4645 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4646
4647 /* Validate. */
4648 if (!(u32Access & X86DESCATTR_UNUSABLE))
4649 {
4650 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4651 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4652 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4655 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4656 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4657 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4658 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4659 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4660 }
4661
4662 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4663 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4664 }
4665
4666 /*
4667 * Guest IDTR.
4668 */
4669 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4670 {
4671 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4672 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4673
4674 /* Validate. */
4675 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4676
4677 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4678 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4679 }
4680
4681 return VINF_SUCCESS;
4682}
4683
4684
4685/**
4686 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4687 * areas. These MSRs will automatically be loaded to the host CPU on every
4688 * successful VM-entry and stored from the host CPU on every successful VM-exit.
4689 *
4690 * This also creates/updates MSR slots for the host MSRs. The actual host
4691 * MSR values are -not- updated here for performance reasons. See
4692 * hmR0VmxSaveHostMsrs().
4693 *
4694 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4695 *
4696 * @returns VBox status code.
4697 * @param pVCpu Pointer to the VMCPU.
4698 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4699 * out-of-sync. Make sure to update the required fields
4700 * before using them.
4701 *
4702 * @remarks No-long-jump zone!!!
4703 */
4704static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4705{
4706 AssertPtr(pVCpu);
4707 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4708
4709 /*
4710 * MSRs that we use the auto-load/store MSR area in the VMCS.
4711 */
4712 PVM pVM = pVCpu->CTX_SUFF(pVM);
4713 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4714 {
4715 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4716#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4717 if (pVM->hm.s.fAllow64BitGuests)
4718 {
4719 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4720 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4721 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4722 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4723# ifdef DEBUG
4724 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4725 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4726 {
4727 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4728 pMsr->u64Value));
4729 }
4730# endif
4731 }
4732#endif
4733 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4734 }
4735
4736 /*
4737 * Guest Sysenter MSRs.
4738 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4739 * VM-exits on WRMSRs for these MSRs.
4740 */
4741 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4742 {
4743 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4744 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4745 }
4746
4747 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4748 {
4749 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4750 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4751 }
4752
4753 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4754 {
4755 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4756 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4757 }
4758
4759 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4760 {
4761 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4762 {
4763 /*
4764 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4765 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4766 */
4767 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4768 {
4769 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4770 AssertRCReturn(rc,rc);
4771 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4772 }
4773 else
4774 {
4775 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4776 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4777 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4778 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4779 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4780 }
4781 }
4782 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4783 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4784 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4785 }
4786
4787 return VINF_SUCCESS;
4788}
4789
4790
4791/**
4792 * Loads the guest activity state into the guest-state area in the VMCS.
4793 *
4794 * @returns VBox status code.
4795 * @param pVCpu Pointer to the VMCPU.
4796 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4797 * out-of-sync. Make sure to update the required fields
4798 * before using them.
4799 *
4800 * @remarks No-long-jump zone!!!
4801 */
4802static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4803{
4804 NOREF(pCtx);
4805 /** @todo See if we can make use of other states, e.g.
4806 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4807 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4808 {
4809 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4810 AssertRCReturn(rc, rc);
4811
4812 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4813 }
4814 return VINF_SUCCESS;
4815}
4816
4817
4818/**
4819 * Sets up the appropriate function to run guest code.
4820 *
4821 * @returns VBox status code.
4822 * @param pVCpu Pointer to the VMCPU.
4823 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4824 * out-of-sync. Make sure to update the required fields
4825 * before using them.
4826 *
4827 * @remarks No-long-jump zone!!!
4828 */
4829static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4830{
4831 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4832 {
4833#ifndef VBOX_ENABLE_64_BITS_GUESTS
4834 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4835#endif
4836 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4837#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4838 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4839 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4840 {
4841 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4842 {
4843 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4844 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4845 | HM_CHANGED_VMX_EXIT_CTLS
4846 | HM_CHANGED_VMX_ENTRY_CTLS
4847 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4848 }
4849 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4850 }
4851#else
4852 /* 64-bit host or hybrid host. */
4853 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4854#endif
4855 }
4856 else
4857 {
4858 /* Guest is not in long mode, use the 32-bit handler. */
4859#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4860 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4861 {
4862 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4863 {
4864 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4865 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4866 | HM_CHANGED_VMX_EXIT_CTLS
4867 | HM_CHANGED_VMX_ENTRY_CTLS
4868 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4869 }
4870 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4871 }
4872#else
4873 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4874#endif
4875 }
4876 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4877 return VINF_SUCCESS;
4878}
4879
4880
4881/**
4882 * Wrapper for running the guest code in VT-x.
4883 *
4884 * @returns VBox strict status code.
4885 * @param pVM Pointer to the VM.
4886 * @param pVCpu Pointer to the VMCPU.
4887 * @param pCtx Pointer to the guest-CPU context.
4888 *
4889 * @remarks No-long-jump zone!!!
4890 */
4891DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4892{
4893 /*
4894 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4895 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4896 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4897 */
4898 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4899 /** @todo Add stats for resume vs launch. */
4900#ifdef VBOX_WITH_KERNEL_USING_XMM
4901 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4902#else
4903 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4904#endif
4905}
4906
4907
4908/**
4909 * Reports world-switch error and dumps some useful debug info.
4910 *
4911 * @param pVM Pointer to the VM.
4912 * @param pVCpu Pointer to the VMCPU.
4913 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4914 * @param pCtx Pointer to the guest-CPU context.
4915 * @param pVmxTransient Pointer to the VMX transient structure (only
4916 * exitReason updated).
4917 */
4918static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4919{
4920 Assert(pVM);
4921 Assert(pVCpu);
4922 Assert(pCtx);
4923 Assert(pVmxTransient);
4924 HMVMX_ASSERT_PREEMPT_SAFE();
4925
4926 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4927 switch (rcVMRun)
4928 {
4929 case VERR_VMX_INVALID_VMXON_PTR:
4930 AssertFailed();
4931 break;
4932 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4933 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4934 {
4935 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4936 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4937 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4938 AssertRC(rc);
4939
4940 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4941 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4942 Cannot do it here as we may have been long preempted. */
4943
4944#ifdef VBOX_STRICT
4945 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4946 pVmxTransient->uExitReason));
4947 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4948 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4949 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4950 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4951 else
4952 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4953 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4954 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4955
4956 /* VMX control bits. */
4957 uint32_t u32Val;
4958 uint64_t u64Val;
4959 HMVMXHCUINTREG uHCReg;
4960 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4961 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4962 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4963 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4964 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4965 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4970 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4971 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4974 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4975 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4976 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4977 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4978 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4979 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4980 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4981 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4982 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4983 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4984 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4985 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4986 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4987 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4992 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4993 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4994 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4995 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4996 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4997 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4998 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4999 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5000 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5001 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5002
5003 /* Guest bits. */
5004 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5005 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5006 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5007 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5008 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5009 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5010 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5011 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5012
5013 /* Host bits. */
5014 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5015 Log4(("Host CR0 %#RHr\n", uHCReg));
5016 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5017 Log4(("Host CR3 %#RHr\n", uHCReg));
5018 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5019 Log4(("Host CR4 %#RHr\n", uHCReg));
5020
5021 RTGDTR HostGdtr;
5022 PCX86DESCHC pDesc;
5023 ASMGetGDTR(&HostGdtr);
5024 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5025 Log4(("Host CS %#08x\n", u32Val));
5026 if (u32Val < HostGdtr.cbGdt)
5027 {
5028 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5029 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5030 }
5031
5032 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5033 Log4(("Host DS %#08x\n", u32Val));
5034 if (u32Val < HostGdtr.cbGdt)
5035 {
5036 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5037 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5038 }
5039
5040 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5041 Log4(("Host ES %#08x\n", u32Val));
5042 if (u32Val < HostGdtr.cbGdt)
5043 {
5044 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5045 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5046 }
5047
5048 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5049 Log4(("Host FS %#08x\n", u32Val));
5050 if (u32Val < HostGdtr.cbGdt)
5051 {
5052 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5053 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5054 }
5055
5056 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5057 Log4(("Host GS %#08x\n", u32Val));
5058 if (u32Val < HostGdtr.cbGdt)
5059 {
5060 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5061 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5062 }
5063
5064 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5065 Log4(("Host SS %#08x\n", u32Val));
5066 if (u32Val < HostGdtr.cbGdt)
5067 {
5068 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5069 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5070 }
5071
5072 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5073 Log4(("Host TR %#08x\n", u32Val));
5074 if (u32Val < HostGdtr.cbGdt)
5075 {
5076 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5077 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5078 }
5079
5080 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5081 Log4(("Host TR Base %#RHv\n", uHCReg));
5082 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5083 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5084 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5085 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5086 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5087 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5088 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5089 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5091 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5092 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5093 Log4(("Host RSP %#RHv\n", uHCReg));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5095 Log4(("Host RIP %#RHv\n", uHCReg));
5096# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5097 if (HMVMX_IS_64BIT_HOST_MODE())
5098 {
5099 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5100 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5101 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5102 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5103 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5104 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5105 }
5106# endif
5107#endif /* VBOX_STRICT */
5108 break;
5109 }
5110
5111 default:
5112 /* Impossible */
5113 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5114 break;
5115 }
5116 NOREF(pVM); NOREF(pCtx);
5117}
5118
5119
5120#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5121#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5122# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5123#endif
5124#ifdef VBOX_STRICT
5125static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5126{
5127 switch (idxField)
5128 {
5129 case VMX_VMCS_GUEST_RIP:
5130 case VMX_VMCS_GUEST_RSP:
5131 case VMX_VMCS_GUEST_SYSENTER_EIP:
5132 case VMX_VMCS_GUEST_SYSENTER_ESP:
5133 case VMX_VMCS_GUEST_GDTR_BASE:
5134 case VMX_VMCS_GUEST_IDTR_BASE:
5135 case VMX_VMCS_GUEST_CS_BASE:
5136 case VMX_VMCS_GUEST_DS_BASE:
5137 case VMX_VMCS_GUEST_ES_BASE:
5138 case VMX_VMCS_GUEST_FS_BASE:
5139 case VMX_VMCS_GUEST_GS_BASE:
5140 case VMX_VMCS_GUEST_SS_BASE:
5141 case VMX_VMCS_GUEST_LDTR_BASE:
5142 case VMX_VMCS_GUEST_TR_BASE:
5143 case VMX_VMCS_GUEST_CR3:
5144 return true;
5145 }
5146 return false;
5147}
5148
5149static bool hmR0VmxIsValidReadField(uint32_t idxField)
5150{
5151 switch (idxField)
5152 {
5153 /* Read-only fields. */
5154 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5155 return true;
5156 }
5157 /* Remaining readable fields should also be writable. */
5158 return hmR0VmxIsValidWriteField(idxField);
5159}
5160#endif /* VBOX_STRICT */
5161
5162
5163/**
5164 * Executes the specified handler in 64-bit mode.
5165 *
5166 * @returns VBox status code.
5167 * @param pVM Pointer to the VM.
5168 * @param pVCpu Pointer to the VMCPU.
5169 * @param pCtx Pointer to the guest CPU context.
5170 * @param enmOp The operation to perform.
5171 * @param cbParam Number of parameters.
5172 * @param paParam Array of 32-bit parameters.
5173 */
5174VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5175 uint32_t *paParam)
5176{
5177 int rc, rc2;
5178 PHMGLOBALCPUINFO pCpu;
5179 RTHCPHYS HCPhysCpuPage;
5180 RTCCUINTREG uOldEflags;
5181
5182 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5183 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5184 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5185 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5186
5187#ifdef VBOX_STRICT
5188 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5189 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5190
5191 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5192 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5193#endif
5194
5195 /* Disable interrupts. */
5196 uOldEflags = ASMIntDisableFlags();
5197
5198#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5199 RTCPUID idHostCpu = RTMpCpuId();
5200 CPUMR0SetLApic(pVCpu, idHostCpu);
5201#endif
5202
5203 pCpu = HMR0GetCurrentCpu();
5204 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5205
5206 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5207 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5208
5209 /* Leave VMX Root Mode. */
5210 VMXDisable();
5211
5212 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5213
5214 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5215 CPUMSetHyperEIP(pVCpu, enmOp);
5216 for (int i = (int)cbParam - 1; i >= 0; i--)
5217 CPUMPushHyper(pVCpu, paParam[i]);
5218
5219 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5220
5221 /* Call the switcher. */
5222 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5223 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5224
5225 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5226 /* Make sure the VMX instructions don't cause #UD faults. */
5227 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5228
5229 /* Re-enter VMX Root Mode */
5230 rc2 = VMXEnable(HCPhysCpuPage);
5231 if (RT_FAILURE(rc2))
5232 {
5233 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5234 ASMSetFlags(uOldEflags);
5235 return rc2;
5236 }
5237
5238 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5239 AssertRC(rc2);
5240 Assert(!(ASMGetFlags() & X86_EFL_IF));
5241 ASMSetFlags(uOldEflags);
5242 return rc;
5243}
5244
5245
5246/**
5247 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5248 * supporting 64-bit guests.
5249 *
5250 * @returns VBox status code.
5251 * @param fResume Whether to VMLAUNCH or VMRESUME.
5252 * @param pCtx Pointer to the guest-CPU context.
5253 * @param pCache Pointer to the VMCS cache.
5254 * @param pVM Pointer to the VM.
5255 * @param pVCpu Pointer to the VMCPU.
5256 */
5257DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5258{
5259 uint32_t aParam[6];
5260 PHMGLOBALCPUINFO pCpu = NULL;
5261 RTHCPHYS HCPhysCpuPage = 0;
5262 int rc = VERR_INTERNAL_ERROR_5;
5263
5264 pCpu = HMR0GetCurrentCpu();
5265 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5266
5267#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5268 pCache->uPos = 1;
5269 pCache->interPD = PGMGetInterPaeCR3(pVM);
5270 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5271#endif
5272
5273#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5274 pCache->TestIn.HCPhysCpuPage = 0;
5275 pCache->TestIn.HCPhysVmcs = 0;
5276 pCache->TestIn.pCache = 0;
5277 pCache->TestOut.HCPhysVmcs = 0;
5278 pCache->TestOut.pCache = 0;
5279 pCache->TestOut.pCtx = 0;
5280 pCache->TestOut.eflags = 0;
5281#endif
5282
5283 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5284 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5285 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5286 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5287 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5288 aParam[5] = 0;
5289
5290#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5291 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5292 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5293#endif
5294 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5295
5296#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5297 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5298 Assert(pCtx->dr[4] == 10);
5299 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5300#endif
5301
5302#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5303 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5304 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5305 pVCpu->hm.s.vmx.HCPhysVmcs));
5306 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5307 pCache->TestOut.HCPhysVmcs));
5308 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5309 pCache->TestOut.pCache));
5310 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5311 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5312 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5313 pCache->TestOut.pCtx));
5314 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5315#endif
5316 return rc;
5317}
5318
5319
5320/**
5321 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5322 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5323 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5324 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5325 *
5326 * @returns VBox status code.
5327 * @param pVM Pointer to the VM.
5328 * @param pVCpu Pointer to the VMCPU.
5329 */
5330static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5331{
5332#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5333{ \
5334 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5335 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5336 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5337 ++cReadFields; \
5338}
5339
5340 AssertPtr(pVM);
5341 AssertPtr(pVCpu);
5342 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5343 uint32_t cReadFields = 0;
5344
5345 /*
5346 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5347 * and serve to indicate exceptions to the rules.
5348 */
5349
5350 /* Guest-natural selector base fields. */
5351#if 0
5352 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5353 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5354 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5355#endif
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5357 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5368#if 0
5369 /* Unused natural width guest-state fields. */
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5371 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5372#endif
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5375
5376 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5377#if 0
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5387#endif
5388
5389 /* Natural width guest-state fields. */
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5391#if 0
5392 /* Currently unused field. */
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5394#endif
5395
5396 if (pVM->hm.s.fNestedPaging)
5397 {
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5399 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5400 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5401 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5402 }
5403 else
5404 {
5405 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5406 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5407 }
5408
5409#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5410 return VINF_SUCCESS;
5411}
5412
5413
5414/**
5415 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5416 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5417 * darwin, running 64-bit guests).
5418 *
5419 * @returns VBox status code.
5420 * @param pVCpu Pointer to the VMCPU.
5421 * @param idxField The VMCS field encoding.
5422 * @param u64Val 16, 32 or 64-bit value.
5423 */
5424VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5425{
5426 int rc;
5427 switch (idxField)
5428 {
5429 /*
5430 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5431 */
5432 /* 64-bit Control fields. */
5433 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5434 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5435 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5436 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5437 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5438 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5439 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5440 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5441 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5442 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5443 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5444 case VMX_VMCS64_CTRL_EPTP_FULL:
5445 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5446 /* 64-bit Guest-state fields. */
5447 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5448 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5449 case VMX_VMCS64_GUEST_PAT_FULL:
5450 case VMX_VMCS64_GUEST_EFER_FULL:
5451 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5452 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5453 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5454 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5455 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5456 /* 64-bit Host-state fields. */
5457 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5458 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5459 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5460 {
5461 rc = VMXWriteVmcs32(idxField, u64Val);
5462 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5463 break;
5464 }
5465
5466 /*
5467 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5468 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5469 */
5470 /* Natural-width Guest-state fields. */
5471 case VMX_VMCS_GUEST_CR3:
5472 case VMX_VMCS_GUEST_ES_BASE:
5473 case VMX_VMCS_GUEST_CS_BASE:
5474 case VMX_VMCS_GUEST_SS_BASE:
5475 case VMX_VMCS_GUEST_DS_BASE:
5476 case VMX_VMCS_GUEST_FS_BASE:
5477 case VMX_VMCS_GUEST_GS_BASE:
5478 case VMX_VMCS_GUEST_LDTR_BASE:
5479 case VMX_VMCS_GUEST_TR_BASE:
5480 case VMX_VMCS_GUEST_GDTR_BASE:
5481 case VMX_VMCS_GUEST_IDTR_BASE:
5482 case VMX_VMCS_GUEST_RSP:
5483 case VMX_VMCS_GUEST_RIP:
5484 case VMX_VMCS_GUEST_SYSENTER_ESP:
5485 case VMX_VMCS_GUEST_SYSENTER_EIP:
5486 {
5487 if (!(u64Val >> 32))
5488 {
5489 /* If this field is 64-bit, VT-x will zero out the top bits. */
5490 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5491 }
5492 else
5493 {
5494 /* Assert that only the 32->64 switcher case should ever come here. */
5495 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5496 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5497 }
5498 break;
5499 }
5500
5501 default:
5502 {
5503 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5504 rc = VERR_INVALID_PARAMETER;
5505 break;
5506 }
5507 }
5508 AssertRCReturn(rc, rc);
5509 return rc;
5510}
5511
5512
5513/**
5514 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5515 * hosts (except darwin) for 64-bit guests.
5516 *
5517 * @param pVCpu Pointer to the VMCPU.
5518 * @param idxField The VMCS field encoding.
5519 * @param u64Val 16, 32 or 64-bit value.
5520 */
5521VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5522{
5523 AssertPtr(pVCpu);
5524 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5525
5526 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5527 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5528
5529 /* Make sure there are no duplicates. */
5530 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5531 {
5532 if (pCache->Write.aField[i] == idxField)
5533 {
5534 pCache->Write.aFieldVal[i] = u64Val;
5535 return VINF_SUCCESS;
5536 }
5537 }
5538
5539 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5540 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5541 pCache->Write.cValidEntries++;
5542 return VINF_SUCCESS;
5543}
5544
5545/* Enable later when the assembly code uses these as callbacks. */
5546#if 0
5547/*
5548 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5549 *
5550 * @param pVCpu Pointer to the VMCPU.
5551 * @param pCache Pointer to the VMCS cache.
5552 *
5553 * @remarks No-long-jump zone!!!
5554 */
5555VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5556{
5557 AssertPtr(pCache);
5558 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5559 {
5560 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5561 AssertRC(rc);
5562 }
5563 pCache->Write.cValidEntries = 0;
5564}
5565
5566
5567/**
5568 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5569 *
5570 * @param pVCpu Pointer to the VMCPU.
5571 * @param pCache Pointer to the VMCS cache.
5572 *
5573 * @remarks No-long-jump zone!!!
5574 */
5575VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5576{
5577 AssertPtr(pCache);
5578 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5579 {
5580 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5581 AssertRC(rc);
5582 }
5583}
5584#endif
5585#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5586
5587
5588/**
5589 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5590 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5591 * timer.
5592 *
5593 * @returns VBox status code.
5594 * @param pVCpu Pointer to the VMCPU.
5595 *
5596 * @remarks No-long-jump zone!!!
5597 */
5598static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5599{
5600 int rc = VERR_INTERNAL_ERROR_5;
5601 bool fOffsettedTsc = false;
5602 bool fParavirtTsc = false;
5603 PVM pVM = pVCpu->CTX_SUFF(pVM);
5604 if (pVM->hm.s.vmx.fUsePreemptTimer)
5605 {
5606 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fOffsettedTsc,
5607 &fParavirtTsc);
5608
5609 /* Make sure the returned values have sane upper and lower boundaries. */
5610 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5611 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5612 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5613 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5614
5615 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5616 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5617 }
5618 else
5619 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5620
5621 /** @todo later optimize this to be done elsewhere and not before every
5622 * VM-entry. */
5623 if (fParavirtTsc)
5624 {
5625 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5626 AssertRC(rc);
5627 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5628 }
5629
5630 if (fOffsettedTsc)
5631 {
5632 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5633 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5634
5635 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5636 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5637 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5638 }
5639 else
5640 {
5641 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5642 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5643 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5644 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5645 }
5646}
5647
5648
5649/**
5650 * Determines if an exception is a contributory exception. Contributory
5651 * exceptions are ones which can cause double-faults. Page-fault is
5652 * intentionally not included here as it's a conditional contributory exception.
5653 *
5654 * @returns true if the exception is contributory, false otherwise.
5655 * @param uVector The exception vector.
5656 */
5657DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5658{
5659 switch (uVector)
5660 {
5661 case X86_XCPT_GP:
5662 case X86_XCPT_SS:
5663 case X86_XCPT_NP:
5664 case X86_XCPT_TS:
5665 case X86_XCPT_DE:
5666 return true;
5667 default:
5668 break;
5669 }
5670 return false;
5671}
5672
5673
5674/**
5675 * Sets an event as a pending event to be injected into the guest.
5676 *
5677 * @param pVCpu Pointer to the VMCPU.
5678 * @param u32IntInfo The VM-entry interruption-information field.
5679 * @param cbInstr The VM-entry instruction length in bytes (for software
5680 * interrupts, exceptions and privileged software
5681 * exceptions).
5682 * @param u32ErrCode The VM-entry exception error code.
5683 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5684 * page-fault.
5685 *
5686 * @remarks Statistics counter assumes this is a guest event being injected or
5687 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5688 * always incremented.
5689 */
5690DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5691 RTGCUINTPTR GCPtrFaultAddress)
5692{
5693 Assert(!pVCpu->hm.s.Event.fPending);
5694 pVCpu->hm.s.Event.fPending = true;
5695 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5696 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5697 pVCpu->hm.s.Event.cbInstr = cbInstr;
5698 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5699
5700 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5701}
5702
5703
5704/**
5705 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5706 *
5707 * @param pVCpu Pointer to the VMCPU.
5708 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5709 * out-of-sync. Make sure to update the required fields
5710 * before using them.
5711 */
5712DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5713{
5714 NOREF(pMixedCtx);
5715 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5716 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5717 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5718 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5719}
5720
5721
5722/**
5723 * Handle a condition that occurred while delivering an event through the guest
5724 * IDT.
5725 *
5726 * @returns VBox status code (informational error codes included).
5727 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5728 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5729 * continue execution of the guest which will delivery the #DF.
5730 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5731 *
5732 * @param pVCpu Pointer to the VMCPU.
5733 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5734 * out-of-sync. Make sure to update the required fields
5735 * before using them.
5736 * @param pVmxTransient Pointer to the VMX transient structure.
5737 *
5738 * @remarks No-long-jump zone!!!
5739 */
5740static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5741{
5742 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5743
5744 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5745 AssertRCReturn(rc, rc);
5746 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5747 AssertRCReturn(rc, rc);
5748
5749 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5750 {
5751 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5752 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5753
5754 typedef enum
5755 {
5756 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5757 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5758 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5759 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5760 } VMXREFLECTXCPT;
5761
5762 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5763 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5764 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5765 {
5766 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5767 {
5768 enmReflect = VMXREFLECTXCPT_XCPT;
5769#ifdef VBOX_STRICT
5770 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5771 && uExitVector == X86_XCPT_PF)
5772 {
5773 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5774 }
5775#endif
5776 if ( uExitVector == X86_XCPT_PF
5777 && uIdtVector == X86_XCPT_PF)
5778 {
5779 pVmxTransient->fVectoringDoublePF = true;
5780 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5781 }
5782 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5783 && hmR0VmxIsContributoryXcpt(uExitVector)
5784 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5785 || uIdtVector == X86_XCPT_PF))
5786 {
5787 enmReflect = VMXREFLECTXCPT_DF;
5788 }
5789 else if (uIdtVector == X86_XCPT_DF)
5790 enmReflect = VMXREFLECTXCPT_TF;
5791 }
5792 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5793 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5794 {
5795 /*
5796 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5797 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5798 */
5799 enmReflect = VMXREFLECTXCPT_XCPT;
5800
5801 if (uExitVector == X86_XCPT_PF)
5802 {
5803 pVmxTransient->fVectoringPF = true;
5804 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5805 }
5806 }
5807 }
5808 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5809 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5810 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5811 {
5812 /*
5813 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5814 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5815 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5816 */
5817 enmReflect = VMXREFLECTXCPT_XCPT;
5818 }
5819
5820 /*
5821 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5822 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5823 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5824 *
5825 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5826 */
5827 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5828 && enmReflect == VMXREFLECTXCPT_XCPT
5829 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5830 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5831 {
5832 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5833 }
5834
5835 switch (enmReflect)
5836 {
5837 case VMXREFLECTXCPT_XCPT:
5838 {
5839 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5840 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5841 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5842
5843 uint32_t u32ErrCode = 0;
5844 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5845 {
5846 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5847 AssertRCReturn(rc, rc);
5848 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5849 }
5850
5851 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5852 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5853 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5854 rc = VINF_SUCCESS;
5855 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5856 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5857
5858 break;
5859 }
5860
5861 case VMXREFLECTXCPT_DF:
5862 {
5863 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5864 rc = VINF_HM_DOUBLE_FAULT;
5865 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5866 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5867
5868 break;
5869 }
5870
5871 case VMXREFLECTXCPT_TF:
5872 {
5873 rc = VINF_EM_RESET;
5874 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5875 uExitVector));
5876 break;
5877 }
5878
5879 default:
5880 Assert(rc == VINF_SUCCESS);
5881 break;
5882 }
5883 }
5884 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5885 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5886 && uExitVector != X86_XCPT_DF
5887 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5888 {
5889 /*
5890 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5891 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5892 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5893 */
5894 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5895 {
5896 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5897 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5898 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5899 }
5900 }
5901
5902 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5903 return rc;
5904}
5905
5906
5907/**
5908 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5909 *
5910 * @returns VBox status code.
5911 * @param pVCpu Pointer to the VMCPU.
5912 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5913 * out-of-sync. Make sure to update the required fields
5914 * before using them.
5915 *
5916 * @remarks No-long-jump zone!!!
5917 */
5918static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5919{
5920 NOREF(pMixedCtx);
5921
5922 /*
5923 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5924 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5925 */
5926 VMMRZCallRing3Disable(pVCpu);
5927 HM_DISABLE_PREEMPT_IF_NEEDED();
5928
5929 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5930 {
5931 uint32_t uVal = 0;
5932 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5933 AssertRCReturn(rc, rc);
5934
5935 uint32_t uShadow = 0;
5936 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5937 AssertRCReturn(rc, rc);
5938
5939 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5940 CPUMSetGuestCR0(pVCpu, uVal);
5941 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5942 }
5943
5944 HM_RESTORE_PREEMPT_IF_NEEDED();
5945 VMMRZCallRing3Enable(pVCpu);
5946 return VINF_SUCCESS;
5947}
5948
5949
5950/**
5951 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5952 *
5953 * @returns VBox status code.
5954 * @param pVCpu Pointer to the VMCPU.
5955 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5956 * out-of-sync. Make sure to update the required fields
5957 * before using them.
5958 *
5959 * @remarks No-long-jump zone!!!
5960 */
5961static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5962{
5963 NOREF(pMixedCtx);
5964
5965 int rc = VINF_SUCCESS;
5966 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5967 {
5968 uint32_t uVal = 0;
5969 uint32_t uShadow = 0;
5970 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5971 AssertRCReturn(rc, rc);
5972 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5973 AssertRCReturn(rc, rc);
5974
5975 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5976 CPUMSetGuestCR4(pVCpu, uVal);
5977 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5978 }
5979 return rc;
5980}
5981
5982
5983/**
5984 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5985 *
5986 * @returns VBox status code.
5987 * @param pVCpu Pointer to the VMCPU.
5988 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5989 * out-of-sync. Make sure to update the required fields
5990 * before using them.
5991 *
5992 * @remarks No-long-jump zone!!!
5993 */
5994static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5995{
5996 int rc = VINF_SUCCESS;
5997 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5998 {
5999 uint64_t u64Val = 0;
6000 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6001 AssertRCReturn(rc, rc);
6002
6003 pMixedCtx->rip = u64Val;
6004 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6005 }
6006 return rc;
6007}
6008
6009
6010/**
6011 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6012 *
6013 * @returns VBox status code.
6014 * @param pVCpu Pointer to the VMCPU.
6015 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6016 * out-of-sync. Make sure to update the required fields
6017 * before using them.
6018 *
6019 * @remarks No-long-jump zone!!!
6020 */
6021static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6022{
6023 int rc = VINF_SUCCESS;
6024 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6025 {
6026 uint64_t u64Val = 0;
6027 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6028 AssertRCReturn(rc, rc);
6029
6030 pMixedCtx->rsp = u64Val;
6031 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6032 }
6033 return rc;
6034}
6035
6036
6037/**
6038 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6039 *
6040 * @returns VBox status code.
6041 * @param pVCpu Pointer to the VMCPU.
6042 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6043 * out-of-sync. Make sure to update the required fields
6044 * before using them.
6045 *
6046 * @remarks No-long-jump zone!!!
6047 */
6048static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6049{
6050 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6051 {
6052 uint32_t uVal = 0;
6053 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6054 AssertRCReturn(rc, rc);
6055
6056 pMixedCtx->eflags.u32 = uVal;
6057 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6058 {
6059 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6060 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6061
6062 pMixedCtx->eflags.Bits.u1VM = 0;
6063 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6064 }
6065
6066 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6067 }
6068 return VINF_SUCCESS;
6069}
6070
6071
6072/**
6073 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6074 * guest-CPU context.
6075 */
6076DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6077{
6078 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6079 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6080 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6081 return rc;
6082}
6083
6084
6085/**
6086 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6087 * from the guest-state area in the VMCS.
6088 *
6089 * @param pVCpu Pointer to the VMCPU.
6090 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6091 * out-of-sync. Make sure to update the required fields
6092 * before using them.
6093 *
6094 * @remarks No-long-jump zone!!!
6095 */
6096static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6097{
6098 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6099 {
6100 uint32_t uIntrState = 0;
6101 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6102 AssertRC(rc);
6103
6104 if (!uIntrState)
6105 {
6106 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6107 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6108
6109 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6110 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6111 }
6112 else
6113 {
6114 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6115 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6116 {
6117 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6118 AssertRC(rc);
6119 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6120 AssertRC(rc);
6121
6122 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6123 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6124 }
6125 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6126 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6127
6128 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6129 {
6130 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6131 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6132 }
6133 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6134 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6135 }
6136
6137 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6138 }
6139}
6140
6141
6142/**
6143 * Saves the guest's activity state.
6144 *
6145 * @returns VBox status code.
6146 * @param pVCpu Pointer to the VMCPU.
6147 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6148 * out-of-sync. Make sure to update the required fields
6149 * before using them.
6150 *
6151 * @remarks No-long-jump zone!!!
6152 */
6153static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6154{
6155 NOREF(pMixedCtx);
6156 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6157 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6158 return VINF_SUCCESS;
6159}
6160
6161
6162/**
6163 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6164 * the current VMCS into the guest-CPU context.
6165 *
6166 * @returns VBox status code.
6167 * @param pVCpu Pointer to the VMCPU.
6168 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6169 * out-of-sync. Make sure to update the required fields
6170 * before using them.
6171 *
6172 * @remarks No-long-jump zone!!!
6173 */
6174static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6175{
6176 int rc = VINF_SUCCESS;
6177 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6178 {
6179 uint32_t u32Val = 0;
6180 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6181 pMixedCtx->SysEnter.cs = u32Val;
6182 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6183 }
6184
6185 uint64_t u64Val = 0;
6186 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6187 {
6188 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6189 pMixedCtx->SysEnter.eip = u64Val;
6190 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6191 }
6192 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6193 {
6194 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6195 pMixedCtx->SysEnter.esp = u64Val;
6196 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6197 }
6198 return rc;
6199}
6200
6201
6202/**
6203 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6204 * the CPU back into the guest-CPU context.
6205 *
6206 * @returns VBox status code.
6207 * @param pVCpu Pointer to the VMCPU.
6208 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6209 * out-of-sync. Make sure to update the required fields
6210 * before using them.
6211 *
6212 * @remarks No-long-jump zone!!!
6213 */
6214static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6215{
6216#if HC_ARCH_BITS == 64
6217 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6218 {
6219 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6220 VMMRZCallRing3Disable(pVCpu);
6221 HM_DISABLE_PREEMPT_IF_NEEDED();
6222
6223 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6224 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6225 {
6226 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6227 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6228 }
6229
6230 HM_RESTORE_PREEMPT_IF_NEEDED();
6231 VMMRZCallRing3Enable(pVCpu);
6232 }
6233 else
6234 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6235#else
6236 NOREF(pMixedCtx);
6237 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6238#endif
6239
6240 return VINF_SUCCESS;
6241}
6242
6243
6244/**
6245 * Saves the auto load/store'd guest MSRs from the current VMCS into
6246 * the guest-CPU context.
6247 *
6248 * @returns VBox status code.
6249 * @param pVCpu Pointer to the VMCPU.
6250 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6251 * out-of-sync. Make sure to update the required fields
6252 * before using them.
6253 *
6254 * @remarks No-long-jump zone!!!
6255 */
6256static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6257{
6258 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6259 return VINF_SUCCESS;
6260
6261 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6262 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6263 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6264 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6265 {
6266 switch (pMsr->u32Msr)
6267 {
6268 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6269 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6270 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6271 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6272 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6273 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6274 break;
6275
6276 default:
6277 {
6278 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6279 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6280 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6281 }
6282 }
6283 }
6284
6285 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6286 return VINF_SUCCESS;
6287}
6288
6289
6290/**
6291 * Saves the guest control registers from the current VMCS into the guest-CPU
6292 * context.
6293 *
6294 * @returns VBox status code.
6295 * @param pVCpu Pointer to the VMCPU.
6296 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6297 * out-of-sync. Make sure to update the required fields
6298 * before using them.
6299 *
6300 * @remarks No-long-jump zone!!!
6301 */
6302static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6303{
6304 /* Guest CR0. Guest FPU. */
6305 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6306 AssertRCReturn(rc, rc);
6307
6308 /* Guest CR4. */
6309 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6310 AssertRCReturn(rc, rc);
6311
6312 /* Guest CR2 - updated always during the world-switch or in #PF. */
6313 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6314 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6315 {
6316 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6317 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6318
6319 PVM pVM = pVCpu->CTX_SUFF(pVM);
6320 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6321 || ( pVM->hm.s.fNestedPaging
6322 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6323 {
6324 uint64_t u64Val = 0;
6325 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6326 if (pMixedCtx->cr3 != u64Val)
6327 {
6328 CPUMSetGuestCR3(pVCpu, u64Val);
6329 if (VMMRZCallRing3IsEnabled(pVCpu))
6330 {
6331 PGMUpdateCR3(pVCpu, u64Val);
6332 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6333 }
6334 else
6335 {
6336 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6337 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6338 }
6339 }
6340
6341 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6342 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6343 {
6344 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6345 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6346 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6347 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6348
6349 if (VMMRZCallRing3IsEnabled(pVCpu))
6350 {
6351 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6352 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6353 }
6354 else
6355 {
6356 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6357 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6358 }
6359 }
6360 }
6361
6362 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6363 }
6364
6365 /*
6366 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6367 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6368 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6369 *
6370 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6371 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6372 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6373 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6374 *
6375 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6376 */
6377 if (VMMRZCallRing3IsEnabled(pVCpu))
6378 {
6379 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6380 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6381
6382 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6383 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6384
6385 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6386 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6387 }
6388
6389 return rc;
6390}
6391
6392
6393/**
6394 * Reads a guest segment register from the current VMCS into the guest-CPU
6395 * context.
6396 *
6397 * @returns VBox status code.
6398 * @param pVCpu Pointer to the VMCPU.
6399 * @param idxSel Index of the selector in the VMCS.
6400 * @param idxLimit Index of the segment limit in the VMCS.
6401 * @param idxBase Index of the segment base in the VMCS.
6402 * @param idxAccess Index of the access rights of the segment in the VMCS.
6403 * @param pSelReg Pointer to the segment selector.
6404 *
6405 * @remarks No-long-jump zone!!!
6406 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6407 * macro as that takes care of whether to read from the VMCS cache or
6408 * not.
6409 */
6410DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6411 PCPUMSELREG pSelReg)
6412{
6413 NOREF(pVCpu);
6414
6415 uint32_t u32Val = 0;
6416 int rc = VMXReadVmcs32(idxSel, &u32Val);
6417 AssertRCReturn(rc, rc);
6418 pSelReg->Sel = (uint16_t)u32Val;
6419 pSelReg->ValidSel = (uint16_t)u32Val;
6420 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6421
6422 rc = VMXReadVmcs32(idxLimit, &u32Val);
6423 AssertRCReturn(rc, rc);
6424 pSelReg->u32Limit = u32Val;
6425
6426 uint64_t u64Val = 0;
6427 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6428 AssertRCReturn(rc, rc);
6429 pSelReg->u64Base = u64Val;
6430
6431 rc = VMXReadVmcs32(idxAccess, &u32Val);
6432 AssertRCReturn(rc, rc);
6433 pSelReg->Attr.u = u32Val;
6434
6435 /*
6436 * If VT-x marks the segment as unusable, most other bits remain undefined:
6437 * - For CS the L, D and G bits have meaning.
6438 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6439 * - For the remaining data segments no bits are defined.
6440 *
6441 * The present bit and the unusable bit has been observed to be set at the
6442 * same time (the selector was supposed to be invalid as we started executing
6443 * a V8086 interrupt in ring-0).
6444 *
6445 * What should be important for the rest of the VBox code, is that the P bit is
6446 * cleared. Some of the other VBox code recognizes the unusable bit, but
6447 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6448 * safe side here, we'll strip off P and other bits we don't care about. If
6449 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6450 *
6451 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6452 */
6453 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6454 {
6455 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6456
6457 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6458 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6459 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6460
6461 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6462#ifdef DEBUG_bird
6463 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6464 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6465 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6466#endif
6467 }
6468 return VINF_SUCCESS;
6469}
6470
6471
6472#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6473# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6474 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6475 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6476#else
6477# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6478 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6479 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6480#endif
6481
6482
6483/**
6484 * Saves the guest segment registers from the current VMCS into the guest-CPU
6485 * context.
6486 *
6487 * @returns VBox status code.
6488 * @param pVCpu Pointer to the VMCPU.
6489 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6490 * out-of-sync. Make sure to update the required fields
6491 * before using them.
6492 *
6493 * @remarks No-long-jump zone!!!
6494 */
6495static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6496{
6497 /* Guest segment registers. */
6498 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6499 {
6500 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6501 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6502 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6503 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6504 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6505 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6506 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6507
6508 /* Restore segment attributes for real-on-v86 mode hack. */
6509 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6510 {
6511 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6512 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6513 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6514 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6515 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6516 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6517 }
6518 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6519 }
6520
6521 return VINF_SUCCESS;
6522}
6523
6524
6525/**
6526 * Saves the guest descriptor table registers and task register from the current
6527 * VMCS into the guest-CPU context.
6528 *
6529 * @returns VBox status code.
6530 * @param pVCpu Pointer to the VMCPU.
6531 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6532 * out-of-sync. Make sure to update the required fields
6533 * before using them.
6534 *
6535 * @remarks No-long-jump zone!!!
6536 */
6537static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6538{
6539 int rc = VINF_SUCCESS;
6540
6541 /* Guest LDTR. */
6542 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6543 {
6544 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6545 AssertRCReturn(rc, rc);
6546 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6547 }
6548
6549 /* Guest GDTR. */
6550 uint64_t u64Val = 0;
6551 uint32_t u32Val = 0;
6552 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6553 {
6554 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6555 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6556 pMixedCtx->gdtr.pGdt = u64Val;
6557 pMixedCtx->gdtr.cbGdt = u32Val;
6558 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6559 }
6560
6561 /* Guest IDTR. */
6562 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6563 {
6564 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6565 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6566 pMixedCtx->idtr.pIdt = u64Val;
6567 pMixedCtx->idtr.cbIdt = u32Val;
6568 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6569 }
6570
6571 /* Guest TR. */
6572 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6573 {
6574 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6575 AssertRCReturn(rc, rc);
6576
6577 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6578 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6579 {
6580 rc = VMXLOCAL_READ_SEG(TR, tr);
6581 AssertRCReturn(rc, rc);
6582 }
6583 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6584 }
6585 return rc;
6586}
6587
6588#undef VMXLOCAL_READ_SEG
6589
6590
6591/**
6592 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6593 * context.
6594 *
6595 * @returns VBox status code.
6596 * @param pVCpu Pointer to the VMCPU.
6597 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6598 * out-of-sync. Make sure to update the required fields
6599 * before using them.
6600 *
6601 * @remarks No-long-jump zone!!!
6602 */
6603static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6604{
6605 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6606 {
6607 if (!pVCpu->hm.s.fUsingHyperDR7)
6608 {
6609 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6610 uint32_t u32Val;
6611 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6612 pMixedCtx->dr[7] = u32Val;
6613 }
6614
6615 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6616 }
6617 return VINF_SUCCESS;
6618}
6619
6620
6621/**
6622 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6623 *
6624 * @returns VBox status code.
6625 * @param pVCpu Pointer to the VMCPU.
6626 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6627 * out-of-sync. Make sure to update the required fields
6628 * before using them.
6629 *
6630 * @remarks No-long-jump zone!!!
6631 */
6632static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6633{
6634 NOREF(pMixedCtx);
6635
6636 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6637 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6638 return VINF_SUCCESS;
6639}
6640
6641
6642/**
6643 * Saves the entire guest state from the currently active VMCS into the
6644 * guest-CPU context. This essentially VMREADs all guest-data.
6645 *
6646 * @returns VBox status code.
6647 * @param pVCpu Pointer to the VMCPU.
6648 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6649 * out-of-sync. Make sure to update the required fields
6650 * before using them.
6651 */
6652static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6653{
6654 Assert(pVCpu);
6655 Assert(pMixedCtx);
6656
6657 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6658 return VINF_SUCCESS;
6659
6660 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6661 again on the ring-3 callback path, there is no real need to. */
6662 if (VMMRZCallRing3IsEnabled(pVCpu))
6663 VMMR0LogFlushDisable(pVCpu);
6664 else
6665 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6666 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6667
6668 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6669 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6670
6671 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6672 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6673
6674 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6675 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6676
6677 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6678 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6679
6680 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6681 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6682
6683 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6684 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6685
6686 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6687 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6688
6689 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6690 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6691
6692 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6693 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6694
6695 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6696 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6697
6698 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6699 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6700
6701 if (VMMRZCallRing3IsEnabled(pVCpu))
6702 VMMR0LogFlushEnable(pVCpu);
6703
6704 return rc;
6705}
6706
6707
6708/**
6709 * Check per-VM and per-VCPU force flag actions that require us to go back to
6710 * ring-3 for one reason or another.
6711 *
6712 * @returns VBox status code (information status code included).
6713 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6714 * ring-3.
6715 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6716 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6717 * interrupts)
6718 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6719 * all EMTs to be in ring-3.
6720 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6721 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6722 * to the EM loop.
6723 *
6724 * @param pVM Pointer to the VM.
6725 * @param pVCpu Pointer to the VMCPU.
6726 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6727 * out-of-sync. Make sure to update the required fields
6728 * before using them.
6729 */
6730static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6731{
6732 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6733
6734 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6735 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6736 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6737 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6738 {
6739 /* We need the control registers now, make sure the guest-CPU context is updated. */
6740 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6741 AssertRCReturn(rc3, rc3);
6742
6743 /* Pending HM CR3 sync. */
6744 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6745 {
6746 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6747 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6748 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6749 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6750 }
6751
6752 /* Pending HM PAE PDPEs. */
6753 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6754 {
6755 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6756 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6757 }
6758
6759 /* Pending PGM C3 sync. */
6760 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6761 {
6762 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6763 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6764 if (rc2 != VINF_SUCCESS)
6765 {
6766 AssertRC(rc2);
6767 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6768 return rc2;
6769 }
6770 }
6771
6772 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6773 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6774 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6775 {
6776 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6777 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6778 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6779 return rc2;
6780 }
6781
6782 /* Pending VM request packets, such as hardware interrupts. */
6783 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6784 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6785 {
6786 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6787 return VINF_EM_PENDING_REQUEST;
6788 }
6789
6790 /* Pending PGM pool flushes. */
6791 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6792 {
6793 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6794 return VINF_PGM_POOL_FLUSH_PENDING;
6795 }
6796
6797 /* Pending DMA requests. */
6798 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6799 {
6800 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6801 return VINF_EM_RAW_TO_R3;
6802 }
6803 }
6804
6805 return VINF_SUCCESS;
6806}
6807
6808
6809/**
6810 * Converts any TRPM trap into a pending HM event. This is typically used when
6811 * entering from ring-3 (not longjmp returns).
6812 *
6813 * @param pVCpu Pointer to the VMCPU.
6814 */
6815static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6816{
6817 Assert(TRPMHasTrap(pVCpu));
6818 Assert(!pVCpu->hm.s.Event.fPending);
6819
6820 uint8_t uVector;
6821 TRPMEVENT enmTrpmEvent;
6822 RTGCUINT uErrCode;
6823 RTGCUINTPTR GCPtrFaultAddress;
6824 uint8_t cbInstr;
6825
6826 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6827 AssertRC(rc);
6828
6829 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6830 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6831 if (enmTrpmEvent == TRPM_TRAP)
6832 {
6833 switch (uVector)
6834 {
6835 case X86_XCPT_NMI:
6836 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6837 break;
6838
6839 case X86_XCPT_BP:
6840 case X86_XCPT_OF:
6841 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6842 break;
6843
6844 case X86_XCPT_PF:
6845 case X86_XCPT_DF:
6846 case X86_XCPT_TS:
6847 case X86_XCPT_NP:
6848 case X86_XCPT_SS:
6849 case X86_XCPT_GP:
6850 case X86_XCPT_AC:
6851 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6852 /* no break! */
6853 default:
6854 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6855 break;
6856 }
6857 }
6858 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6859 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6860 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6861 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6862 else
6863 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6864
6865 rc = TRPMResetTrap(pVCpu);
6866 AssertRC(rc);
6867 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6868 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6869
6870 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6871 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6872}
6873
6874
6875/**
6876 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6877 * VT-x to execute any instruction.
6878 *
6879 * @param pvCpu Pointer to the VMCPU.
6880 */
6881static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6882{
6883 Assert(pVCpu->hm.s.Event.fPending);
6884
6885 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6886 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6887 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6888 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6889
6890 /* If a trap was already pending, we did something wrong! */
6891 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6892
6893 TRPMEVENT enmTrapType;
6894 switch (uVectorType)
6895 {
6896 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6897 enmTrapType = TRPM_HARDWARE_INT;
6898 break;
6899
6900 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6901 enmTrapType = TRPM_SOFTWARE_INT;
6902 break;
6903
6904 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6905 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6906 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6907 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6908 enmTrapType = TRPM_TRAP;
6909 break;
6910
6911 default:
6912 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6913 enmTrapType = TRPM_32BIT_HACK;
6914 break;
6915 }
6916
6917 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6918
6919 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6920 AssertRC(rc);
6921
6922 if (fErrorCodeValid)
6923 TRPMSetErrorCode(pVCpu, uErrorCode);
6924
6925 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6926 && uVector == X86_XCPT_PF)
6927 {
6928 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6929 }
6930 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6931 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6932 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6933 {
6934 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6935 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6936 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6937 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6938 }
6939 pVCpu->hm.s.Event.fPending = false;
6940}
6941
6942
6943/**
6944 * Does the necessary state syncing before returning to ring-3 for any reason
6945 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6946 *
6947 * @returns VBox status code.
6948 * @param pVM Pointer to the VM.
6949 * @param pVCpu Pointer to the VMCPU.
6950 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6951 * be out-of-sync. Make sure to update the required
6952 * fields before using them.
6953 * @param fSaveGuestState Whether to save the guest state or not.
6954 *
6955 * @remarks No-long-jmp zone!!!
6956 */
6957static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6958{
6959 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6960 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6961
6962 RTCPUID idCpu = RTMpCpuId();
6963 Log4Func(("HostCpuId=%u\n", idCpu));
6964
6965 /*
6966 * !!! IMPORTANT !!!
6967 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6968 */
6969
6970 /* Save the guest state if necessary. */
6971 if ( fSaveGuestState
6972 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6973 {
6974 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6975 AssertRCReturn(rc, rc);
6976 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6977 }
6978
6979 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6980 if (CPUMIsGuestFPUStateActive(pVCpu))
6981 {
6982 /* We shouldn't reload CR0 without saving it first. */
6983 if (!fSaveGuestState)
6984 {
6985 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6986 AssertRCReturn(rc, rc);
6987 }
6988 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6989 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6990 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6991 }
6992
6993 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6994#ifdef VBOX_STRICT
6995 if (CPUMIsHyperDebugStateActive(pVCpu))
6996 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6997#endif
6998 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6999 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7000 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7001 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7002
7003#if HC_ARCH_BITS == 64
7004 /* Restore host-state bits that VT-x only restores partially. */
7005 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7006 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7007 {
7008 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7009 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7010 }
7011 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7012#endif
7013
7014#if HC_ARCH_BITS == 64
7015 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7016 if ( pVM->hm.s.fAllow64BitGuests
7017 && pVCpu->hm.s.vmx.fLazyMsrs)
7018 {
7019 /* We shouldn't reload the guest MSRs without saving it first. */
7020 if (!fSaveGuestState)
7021 {
7022 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7023 AssertRCReturn(rc, rc);
7024 }
7025 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7026 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7027 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7028 }
7029#endif
7030
7031 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7032 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7033
7034 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7035 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7036 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7037 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7038 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7039 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7040 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7041 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7042
7043 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7044
7045 /** @todo This partially defeats the purpose of having preemption hooks.
7046 * The problem is, deregistering the hooks should be moved to a place that
7047 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7048 * context.
7049 */
7050 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7051 {
7052 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7053 AssertRCReturn(rc, rc);
7054
7055 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7056 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7057 }
7058 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7059 NOREF(idCpu);
7060
7061 return VINF_SUCCESS;
7062}
7063
7064
7065/**
7066 * Leaves the VT-x session.
7067 *
7068 * @returns VBox status code.
7069 * @param pVM Pointer to the VM.
7070 * @param pVCpu Pointer to the VMCPU.
7071 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7072 * out-of-sync. Make sure to update the required fields
7073 * before using them.
7074 *
7075 * @remarks No-long-jmp zone!!!
7076 */
7077DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7078{
7079 HM_DISABLE_PREEMPT_IF_NEEDED();
7080 HMVMX_ASSERT_CPU_SAFE();
7081 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7082 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7083
7084 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7085 and done this from the VMXR0ThreadCtxCallback(). */
7086 if (!pVCpu->hm.s.fLeaveDone)
7087 {
7088 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7089 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7090 pVCpu->hm.s.fLeaveDone = true;
7091 }
7092 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7093
7094 /*
7095 * !!! IMPORTANT !!!
7096 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7097 */
7098
7099 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7100 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7101 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7102 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7103 VMMR0ThreadCtxHooksDeregister(pVCpu);
7104
7105 /* Leave HM context. This takes care of local init (term). */
7106 int rc = HMR0LeaveCpu(pVCpu);
7107
7108 HM_RESTORE_PREEMPT_IF_NEEDED();
7109
7110 return rc;
7111}
7112
7113
7114/**
7115 * Does the necessary state syncing before doing a longjmp to ring-3.
7116 *
7117 * @returns VBox status code.
7118 * @param pVM Pointer to the VM.
7119 * @param pVCpu Pointer to the VMCPU.
7120 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7121 * out-of-sync. Make sure to update the required fields
7122 * before using them.
7123 *
7124 * @remarks No-long-jmp zone!!!
7125 */
7126DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7127{
7128 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7129}
7130
7131
7132/**
7133 * Take necessary actions before going back to ring-3.
7134 *
7135 * An action requires us to go back to ring-3. This function does the necessary
7136 * steps before we can safely return to ring-3. This is not the same as longjmps
7137 * to ring-3, this is voluntary and prepares the guest so it may continue
7138 * executing outside HM (recompiler/IEM).
7139 *
7140 * @returns VBox status code.
7141 * @param pVM Pointer to the VM.
7142 * @param pVCpu Pointer to the VMCPU.
7143 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7144 * out-of-sync. Make sure to update the required fields
7145 * before using them.
7146 * @param rcExit The reason for exiting to ring-3. Can be
7147 * VINF_VMM_UNKNOWN_RING3_CALL.
7148 */
7149static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7150{
7151 Assert(pVM);
7152 Assert(pVCpu);
7153 Assert(pMixedCtx);
7154 HMVMX_ASSERT_PREEMPT_SAFE();
7155
7156 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7157 {
7158 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7159 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7160 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7161 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7162 }
7163
7164 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7165 VMMRZCallRing3Disable(pVCpu);
7166 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7167
7168 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7169 if (pVCpu->hm.s.Event.fPending)
7170 {
7171 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7172 Assert(!pVCpu->hm.s.Event.fPending);
7173 }
7174
7175 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7176 and if we're injecting an event we should have a TRPM trap pending. */
7177 Assert(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu));
7178 Assert(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu));
7179
7180 /* Save guest state and restore host state bits. */
7181 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7182 AssertRCReturn(rc, rc);
7183 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7184 /* Thread-context hooks are unregistered at this point!!! */
7185
7186 /* Sync recompiler state. */
7187 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7188 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7189 | CPUM_CHANGED_LDTR
7190 | CPUM_CHANGED_GDTR
7191 | CPUM_CHANGED_IDTR
7192 | CPUM_CHANGED_TR
7193 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7194 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7195 if ( pVM->hm.s.fNestedPaging
7196 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7197 {
7198 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7199 }
7200
7201 Assert(!pVCpu->hm.s.fClearTrapFlag);
7202
7203 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7204 if (rcExit != VINF_EM_RAW_INTERRUPT)
7205 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7206
7207 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7208
7209 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7210 VMMRZCallRing3RemoveNotification(pVCpu);
7211 VMMRZCallRing3Enable(pVCpu);
7212
7213 return rc;
7214}
7215
7216
7217/**
7218 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7219 * longjump to ring-3 and possibly get preempted.
7220 *
7221 * @returns VBox status code.
7222 * @param pVCpu Pointer to the VMCPU.
7223 * @param enmOperation The operation causing the ring-3 longjump.
7224 * @param pvUser Opaque pointer to the guest-CPU context. The data
7225 * may be out-of-sync. Make sure to update the required
7226 * fields before using them.
7227 */
7228DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7229{
7230 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7231 {
7232 /*
7233 * !!! IMPORTANT !!!
7234 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7235 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7236 */
7237 VMMRZCallRing3RemoveNotification(pVCpu);
7238 VMMRZCallRing3Disable(pVCpu);
7239 HM_DISABLE_PREEMPT_IF_NEEDED();
7240
7241 PVM pVM = pVCpu->CTX_SUFF(pVM);
7242 if (CPUMIsGuestFPUStateActive(pVCpu))
7243 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7244
7245 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7246
7247#if HC_ARCH_BITS == 64
7248 /* Restore host-state bits that VT-x only restores partially. */
7249 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7250 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7251 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7252 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7253
7254 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7255 if ( pVM->hm.s.fAllow64BitGuests
7256 && pVCpu->hm.s.vmx.fLazyMsrs)
7257 {
7258 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7259 }
7260#endif
7261 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7262 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7263 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7264 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7265 {
7266 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7267 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7268 }
7269
7270 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7271 VMMR0ThreadCtxHooksDeregister(pVCpu);
7272
7273 HMR0LeaveCpu(pVCpu);
7274 HM_RESTORE_PREEMPT_IF_NEEDED();
7275 return VINF_SUCCESS;
7276 }
7277
7278 Assert(pVCpu);
7279 Assert(pvUser);
7280 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7281 HMVMX_ASSERT_PREEMPT_SAFE();
7282
7283 VMMRZCallRing3Disable(pVCpu);
7284 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7285
7286 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7287 enmOperation));
7288
7289 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7290 AssertRCReturn(rc, rc);
7291
7292 VMMRZCallRing3Enable(pVCpu);
7293 return VINF_SUCCESS;
7294}
7295
7296
7297/**
7298 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7299 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7300 *
7301 * @param pVCpu Pointer to the VMCPU.
7302 */
7303DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7304{
7305 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7306 {
7307 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7308 {
7309 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7310 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7311 AssertRC(rc);
7312 Log4(("Setup interrupt-window exiting\n"));
7313 }
7314 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7315}
7316
7317
7318/**
7319 * Clears the interrupt-window exiting control in the VMCS.
7320 *
7321 * @param pVCpu Pointer to the VMCPU.
7322 */
7323DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7324{
7325 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7326 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7327 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7328 AssertRC(rc);
7329 Log4(("Cleared interrupt-window exiting\n"));
7330}
7331
7332
7333/**
7334 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7335 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7336 *
7337 * @param pVCpu Pointer to the VMCPU.
7338 */
7339DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7340{
7341 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7342 {
7343 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7344 {
7345 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7346 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7347 AssertRC(rc);
7348 Log4(("Setup NMI-window exiting\n"));
7349 }
7350 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7351}
7352
7353
7354/**
7355 * Clears the NMI-window exiting control in the VMCS.
7356 *
7357 * @param pVCpu Pointer to the VMCPU.
7358 */
7359DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7360{
7361 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7362 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7363 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7364 AssertRC(rc);
7365 Log4(("Cleared NMI-window exiting\n"));
7366}
7367
7368
7369/**
7370 * Evaluates the event to be delivered to the guest and sets it as the pending
7371 * event.
7372 *
7373 * @param pVCpu Pointer to the VMCPU.
7374 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7375 * out-of-sync. Make sure to update the required fields
7376 * before using them.
7377 */
7378static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7379{
7380 Assert(!pVCpu->hm.s.Event.fPending);
7381
7382 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7383 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7384 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7385 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7386 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7387
7388 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7389 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7390 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7391 Assert(!TRPMHasTrap(pVCpu));
7392
7393 /*
7394 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7395 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7396 */
7397 /** @todo SMI. SMIs take priority over NMIs. */
7398 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7399 {
7400 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7401 if ( !fBlockNmi
7402 && !fBlockSti
7403 && !fBlockMovSS)
7404 {
7405 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7406 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7407 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7408
7409 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7410 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7411 }
7412 else
7413 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7414 }
7415 /*
7416 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7417 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7418 */
7419 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7420 && !pVCpu->hm.s.fSingleInstruction)
7421 {
7422 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7423 AssertRC(rc);
7424 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7425 if ( !fBlockInt
7426 && !fBlockSti
7427 && !fBlockMovSS)
7428 {
7429 uint8_t u8Interrupt;
7430 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7431 if (RT_SUCCESS(rc))
7432 {
7433 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7434 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7435 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7436
7437 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7438 }
7439 else
7440 {
7441 /** @todo Does this actually happen? If not turn it into an assertion. */
7442 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7443 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7444 }
7445 }
7446 else
7447 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7448 }
7449}
7450
7451
7452/**
7453 * Sets a pending-debug exception to be delivered to the guest if the guest is
7454 * single-stepping.
7455 *
7456 * @param pVCpu Pointer to the VMCPU.
7457 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7458 * out-of-sync. Make sure to update the required fields
7459 * before using them.
7460 */
7461DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7462{
7463 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7464 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7465 {
7466 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7467 AssertRC(rc);
7468 }
7469}
7470
7471
7472/**
7473 * Injects any pending events into the guest if the guest is in a state to
7474 * receive them.
7475 *
7476 * @returns VBox status code (informational status codes included).
7477 * @param pVCpu Pointer to the VMCPU.
7478 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7479 * out-of-sync. Make sure to update the required fields
7480 * before using them.
7481 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7482 * return VINF_EM_DBG_STEPPED if the event was
7483 * dispatched directly.
7484 */
7485static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7486{
7487 HMVMX_ASSERT_PREEMPT_SAFE();
7488 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7489
7490 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7491 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7492 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7493 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7494
7495 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7496 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7497 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7498 Assert(!TRPMHasTrap(pVCpu));
7499
7500 int rc = VINF_SUCCESS;
7501 if (pVCpu->hm.s.Event.fPending)
7502 {
7503 /*
7504 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7505 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7506 * ended up enabling interrupts outside VT-x.
7507 */
7508 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7509 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7510 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7511 {
7512 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7513 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7514 }
7515
7516#ifdef VBOX_STRICT
7517 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7518 {
7519 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7520 Assert(!fBlockInt);
7521 Assert(!fBlockSti);
7522 Assert(!fBlockMovSS);
7523 }
7524 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7525 {
7526 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7527 Assert(!fBlockSti);
7528 Assert(!fBlockMovSS);
7529 Assert(!fBlockNmi);
7530 }
7531#endif
7532 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7533 (uint8_t)uIntType));
7534 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7535 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7536 AssertRCReturn(rc, rc);
7537
7538 /* Update the interruptibility-state as it could have been changed by
7539 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7540 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7541 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7542
7543#ifdef VBOX_WITH_STATISTICS
7544 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7545 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7546 else
7547 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7548#endif
7549 }
7550
7551 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7552 if ( fBlockSti
7553 || fBlockMovSS)
7554 {
7555 if ( !pVCpu->hm.s.fSingleInstruction
7556 && !DBGFIsStepping(pVCpu))
7557 {
7558 /*
7559 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7560 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7561 * See Intel spec. 27.3.4 "Saving Non-Register State".
7562 */
7563 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7564 AssertRCReturn(rc2, rc2);
7565 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7566 }
7567 else if (pMixedCtx->eflags.Bits.u1TF)
7568 {
7569 /*
7570 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7571 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7572 */
7573 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7574 uIntrState = 0;
7575 }
7576 }
7577
7578 /*
7579 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7580 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7581 */
7582 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7583 AssertRC(rc2);
7584
7585 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7586 NOREF(fBlockMovSS); NOREF(fBlockSti);
7587 return rc;
7588}
7589
7590
7591/**
7592 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7593 *
7594 * @param pVCpu Pointer to the VMCPU.
7595 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7596 * out-of-sync. Make sure to update the required fields
7597 * before using them.
7598 */
7599DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7600{
7601 NOREF(pMixedCtx);
7602 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7603 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7604}
7605
7606
7607/**
7608 * Injects a double-fault (#DF) exception into the VM.
7609 *
7610 * @returns VBox status code (informational status code included).
7611 * @param pVCpu Pointer to the VMCPU.
7612 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7613 * out-of-sync. Make sure to update the required fields
7614 * before using them.
7615 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7616 * and should return VINF_EM_DBG_STEPPED if the event
7617 * is injected directly (register modified by us, not
7618 * by hardware on VM-entry).
7619 * @param puIntrState Pointer to the current guest interruptibility-state.
7620 * This interruptibility-state will be updated if
7621 * necessary. This cannot not be NULL.
7622 */
7623DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7624{
7625 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7626 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7627 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7628 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7629 fStepping, puIntrState);
7630}
7631
7632
7633/**
7634 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7635 *
7636 * @param pVCpu Pointer to the VMCPU.
7637 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7638 * out-of-sync. Make sure to update the required fields
7639 * before using them.
7640 */
7641DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7642{
7643 NOREF(pMixedCtx);
7644 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7645 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7646 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7647}
7648
7649
7650/**
7651 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7652 *
7653 * @param pVCpu Pointer to the VMCPU.
7654 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7655 * out-of-sync. Make sure to update the required fields
7656 * before using them.
7657 * @param cbInstr The value of RIP that is to be pushed on the guest
7658 * stack.
7659 */
7660DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7661{
7662 NOREF(pMixedCtx);
7663 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7664 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7665 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7666}
7667
7668
7669/**
7670 * Injects a general-protection (#GP) fault into the VM.
7671 *
7672 * @returns VBox status code (informational status code included).
7673 * @param pVCpu Pointer to the VMCPU.
7674 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7675 * out-of-sync. Make sure to update the required fields
7676 * before using them.
7677 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7678 * mode, i.e. in real-mode it's not valid).
7679 * @param u32ErrorCode The error code associated with the #GP.
7680 * @param fStepping Whether we're running in
7681 * hmR0VmxRunGuestCodeStep() and should return
7682 * VINF_EM_DBG_STEPPED if the event is injected
7683 * directly (register modified by us, not by
7684 * hardware on VM-entry).
7685 * @param puIntrState Pointer to the current guest interruptibility-state.
7686 * This interruptibility-state will be updated if
7687 * necessary. This cannot not be NULL.
7688 */
7689DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7690 bool fStepping, uint32_t *puIntrState)
7691{
7692 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7693 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7694 if (fErrorCodeValid)
7695 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7696 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7697 fStepping, puIntrState);
7698}
7699
7700
7701/**
7702 * Sets a general-protection (#GP) exception as pending-for-injection into the
7703 * VM.
7704 *
7705 * @param pVCpu Pointer to the VMCPU.
7706 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7707 * out-of-sync. Make sure to update the required fields
7708 * before using them.
7709 * @param u32ErrorCode The error code associated with the #GP.
7710 */
7711DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7712{
7713 NOREF(pMixedCtx);
7714 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7715 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7716 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7717 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7718}
7719
7720
7721/**
7722 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7723 *
7724 * @param pVCpu Pointer to the VMCPU.
7725 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7726 * out-of-sync. Make sure to update the required fields
7727 * before using them.
7728 * @param uVector The software interrupt vector number.
7729 * @param cbInstr The value of RIP that is to be pushed on the guest
7730 * stack.
7731 */
7732DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7733{
7734 NOREF(pMixedCtx);
7735 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7736 if ( uVector == X86_XCPT_BP
7737 || uVector == X86_XCPT_OF)
7738 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7739 else
7740 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7741 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7742}
7743
7744
7745/**
7746 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7747 * stack.
7748 *
7749 * @returns VBox status code (information status code included).
7750 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7751 * @param pVM Pointer to the VM.
7752 * @param pMixedCtx Pointer to the guest-CPU context.
7753 * @param uValue The value to push to the guest stack.
7754 */
7755DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7756{
7757 /*
7758 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7759 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7760 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7761 */
7762 if (pMixedCtx->sp == 1)
7763 return VINF_EM_RESET;
7764 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7765 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7766 AssertRCReturn(rc, rc);
7767 return rc;
7768}
7769
7770
7771/**
7772 * Injects an event into the guest upon VM-entry by updating the relevant fields
7773 * in the VM-entry area in the VMCS.
7774 *
7775 * @returns VBox status code (informational error codes included).
7776 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7777 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7778 *
7779 * @param pVCpu Pointer to the VMCPU.
7780 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7781 * be out-of-sync. Make sure to update the required
7782 * fields before using them.
7783 * @param u64IntInfo The VM-entry interruption-information field.
7784 * @param cbInstr The VM-entry instruction length in bytes (for
7785 * software interrupts, exceptions and privileged
7786 * software exceptions).
7787 * @param u32ErrCode The VM-entry exception error code.
7788 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7789 * @param puIntrState Pointer to the current guest interruptibility-state.
7790 * This interruptibility-state will be updated if
7791 * necessary. This cannot not be NULL.
7792 * @param fStepping Whether we're running in
7793 * hmR0VmxRunGuestCodeStep() and should return
7794 * VINF_EM_DBG_STEPPED if the event is injected
7795 * directly (register modified by us, not by
7796 * hardware on VM-entry).
7797 *
7798 * @remarks Requires CR0!
7799 * @remarks No-long-jump zone!!!
7800 */
7801static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7802 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7803{
7804 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7805 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7806 Assert(puIntrState);
7807 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7808
7809 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7810 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7811
7812#ifdef VBOX_STRICT
7813 /* Validate the error-code-valid bit for hardware exceptions. */
7814 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7815 {
7816 switch (uVector)
7817 {
7818 case X86_XCPT_PF:
7819 case X86_XCPT_DF:
7820 case X86_XCPT_TS:
7821 case X86_XCPT_NP:
7822 case X86_XCPT_SS:
7823 case X86_XCPT_GP:
7824 case X86_XCPT_AC:
7825 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7826 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7827 /* fallthru */
7828 default:
7829 break;
7830 }
7831 }
7832#endif
7833
7834 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7835 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7836 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7837
7838 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7839
7840 /* We require CR0 to check if the guest is in real-mode. */
7841 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7842 AssertRCReturn(rc, rc);
7843
7844 /*
7845 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7846 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7847 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7848 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7849 */
7850 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7851 {
7852 PVM pVM = pVCpu->CTX_SUFF(pVM);
7853 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7854 {
7855 Assert(PDMVmmDevHeapIsEnabled(pVM));
7856 Assert(pVM->hm.s.vmx.pRealModeTSS);
7857
7858 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7859 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7860 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7861 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7862 AssertRCReturn(rc, rc);
7863 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7864
7865 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7866 size_t const cbIdtEntry = sizeof(X86IDTR16);
7867 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7868 {
7869 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7870 if (uVector == X86_XCPT_DF)
7871 return VINF_EM_RESET;
7872
7873 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7874 if (uVector == X86_XCPT_GP)
7875 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7876
7877 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7878 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7879 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7880 fStepping, puIntrState);
7881 }
7882
7883 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7884 uint16_t uGuestIp = pMixedCtx->ip;
7885 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7886 {
7887 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7888 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7889 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7890 }
7891 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7892 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7893
7894 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7895 X86IDTR16 IdtEntry;
7896 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7897 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7898 AssertRCReturn(rc, rc);
7899
7900 /* Construct the stack frame for the interrupt/exception handler. */
7901 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7902 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7903 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7904 AssertRCReturn(rc, rc);
7905
7906 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7907 if (rc == VINF_SUCCESS)
7908 {
7909 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7910 pMixedCtx->rip = IdtEntry.offSel;
7911 pMixedCtx->cs.Sel = IdtEntry.uSel;
7912 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7913 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7914 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7915 && uVector == X86_XCPT_PF)
7916 pMixedCtx->cr2 = GCPtrFaultAddress;
7917
7918 /* If any other guest-state bits are changed here, make sure to update
7919 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7920 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7921 | HM_CHANGED_GUEST_RIP
7922 | HM_CHANGED_GUEST_RFLAGS
7923 | HM_CHANGED_GUEST_RSP);
7924
7925 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7926 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7927 {
7928 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7929 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7930 Log4(("Clearing inhibition due to STI.\n"));
7931 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7932 }
7933 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7934 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7935
7936 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7937 it, if we are returning to ring-3 before executing guest code. */
7938 pVCpu->hm.s.Event.fPending = false;
7939
7940 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7941 if (fStepping)
7942 rc = VINF_EM_DBG_STEPPED;
7943 }
7944 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7945 return rc;
7946 }
7947
7948 /*
7949 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7950 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7951 */
7952 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7953 }
7954
7955 /* Validate. */
7956 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7957 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7958 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7959
7960 /* Inject. */
7961 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7962 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7963 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7964 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7965
7966 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7967 && uVector == X86_XCPT_PF)
7968 pMixedCtx->cr2 = GCPtrFaultAddress;
7969
7970 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7971 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7972
7973 AssertRCReturn(rc, rc);
7974 return rc;
7975}
7976
7977
7978/**
7979 * Clears the interrupt-window exiting control in the VMCS and if necessary
7980 * clears the current event in the VMCS as well.
7981 *
7982 * @returns VBox status code.
7983 * @param pVCpu Pointer to the VMCPU.
7984 *
7985 * @remarks Use this function only to clear events that have not yet been
7986 * delivered to the guest but are injected in the VMCS!
7987 * @remarks No-long-jump zone!!!
7988 */
7989static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7990{
7991 int rc;
7992 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7993
7994 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7995 {
7996 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7997 Assert(!pVCpu->hm.s.Event.fPending);
7998 }
7999
8000 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8001 {
8002 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8003 Assert(!pVCpu->hm.s.Event.fPending);
8004 }
8005
8006 if (!pVCpu->hm.s.Event.fPending)
8007 return;
8008
8009#ifdef VBOX_STRICT
8010 uint32_t u32EntryInfo;
8011 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8012 AssertRC(rc);
8013 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8014#endif
8015
8016 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8017 AssertRC(rc);
8018
8019 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8020 AssertRC(rc);
8021
8022 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8023 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8024}
8025
8026
8027/**
8028 * Enters the VT-x session.
8029 *
8030 * @returns VBox status code.
8031 * @param pVM Pointer to the VM.
8032 * @param pVCpu Pointer to the VMCPU.
8033 * @param pCpu Pointer to the CPU info struct.
8034 */
8035VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8036{
8037 AssertPtr(pVM);
8038 AssertPtr(pVCpu);
8039 Assert(pVM->hm.s.vmx.fSupported);
8040 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8041 NOREF(pCpu); NOREF(pVM);
8042
8043 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8044 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8045
8046#ifdef VBOX_STRICT
8047 /* Make sure we're in VMX root mode. */
8048 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8049 if (!(u32HostCR4 & X86_CR4_VMXE))
8050 {
8051 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8052 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8053 }
8054#endif
8055
8056 /*
8057 * Load the VCPU's VMCS as the current (and active) one.
8058 */
8059 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8060 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8061 if (RT_FAILURE(rc))
8062 return rc;
8063
8064 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8065 pVCpu->hm.s.fLeaveDone = false;
8066 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8067
8068 return VINF_SUCCESS;
8069}
8070
8071
8072/**
8073 * The thread-context callback (only on platforms which support it).
8074 *
8075 * @param enmEvent The thread-context event.
8076 * @param pVCpu Pointer to the VMCPU.
8077 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8078 * @thread EMT(pVCpu)
8079 */
8080VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8081{
8082 NOREF(fGlobalInit);
8083
8084 switch (enmEvent)
8085 {
8086 case RTTHREADCTXEVENT_PREEMPTING:
8087 {
8088 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8089 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8090 VMCPU_ASSERT_EMT(pVCpu);
8091
8092 PVM pVM = pVCpu->CTX_SUFF(pVM);
8093 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8094
8095 /* No longjmps (logger flushes, locks) in this fragile context. */
8096 VMMRZCallRing3Disable(pVCpu);
8097 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8098
8099 /*
8100 * Restore host-state (FPU, debug etc.)
8101 */
8102 if (!pVCpu->hm.s.fLeaveDone)
8103 {
8104 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8105 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8106 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8107 pVCpu->hm.s.fLeaveDone = true;
8108 }
8109
8110 /* Leave HM context, takes care of local init (term). */
8111 int rc = HMR0LeaveCpu(pVCpu);
8112 AssertRC(rc); NOREF(rc);
8113
8114 /* Restore longjmp state. */
8115 VMMRZCallRing3Enable(pVCpu);
8116 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8117 break;
8118 }
8119
8120 case RTTHREADCTXEVENT_RESUMED:
8121 {
8122 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8123 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8124 VMCPU_ASSERT_EMT(pVCpu);
8125
8126 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8127 VMMRZCallRing3Disable(pVCpu);
8128 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8129
8130 /* Initialize the bare minimum state required for HM. This takes care of
8131 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8132 int rc = HMR0EnterCpu(pVCpu);
8133 AssertRC(rc);
8134 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8135
8136 /* Load the active VMCS as the current one. */
8137 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8138 {
8139 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8140 AssertRC(rc); NOREF(rc);
8141 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8142 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8143 }
8144 pVCpu->hm.s.fLeaveDone = false;
8145
8146 /* Restore longjmp state. */
8147 VMMRZCallRing3Enable(pVCpu);
8148 break;
8149 }
8150
8151 default:
8152 break;
8153 }
8154}
8155
8156
8157/**
8158 * Saves the host state in the VMCS host-state.
8159 * Sets up the VM-exit MSR-load area.
8160 *
8161 * The CPU state will be loaded from these fields on every successful VM-exit.
8162 *
8163 * @returns VBox status code.
8164 * @param pVM Pointer to the VM.
8165 * @param pVCpu Pointer to the VMCPU.
8166 *
8167 * @remarks No-long-jump zone!!!
8168 */
8169static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8170{
8171 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8172
8173 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8174 return VINF_SUCCESS;
8175
8176 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8177 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8178
8179 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8180 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8181
8182 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8183 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8184
8185 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8186 return rc;
8187}
8188
8189
8190/**
8191 * Saves the host state in the VMCS host-state.
8192 *
8193 * @returns VBox status code.
8194 * @param pVM Pointer to the VM.
8195 * @param pVCpu Pointer to the VMCPU.
8196 *
8197 * @remarks No-long-jump zone!!!
8198 */
8199VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8200{
8201 AssertPtr(pVM);
8202 AssertPtr(pVCpu);
8203
8204 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8205
8206 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8207 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8208 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8209 return hmR0VmxSaveHostState(pVM, pVCpu);
8210}
8211
8212
8213/**
8214 * Loads the guest state into the VMCS guest-state area.
8215 *
8216 * The will typically be done before VM-entry when the guest-CPU state and the
8217 * VMCS state may potentially be out of sync.
8218 *
8219 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8220 * VM-entry controls.
8221 * Sets up the appropriate VMX non-root function to execute guest code based on
8222 * the guest CPU mode.
8223 *
8224 * @returns VBox status code.
8225 * @param pVM Pointer to the VM.
8226 * @param pVCpu Pointer to the VMCPU.
8227 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8228 * out-of-sync. Make sure to update the required fields
8229 * before using them.
8230 *
8231 * @remarks No-long-jump zone!!!
8232 */
8233static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8234{
8235 AssertPtr(pVM);
8236 AssertPtr(pVCpu);
8237 AssertPtr(pMixedCtx);
8238 HMVMX_ASSERT_PREEMPT_SAFE();
8239
8240 VMMRZCallRing3Disable(pVCpu);
8241 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8242
8243 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8244
8245 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8246
8247 /* Determine real-on-v86 mode. */
8248 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8249 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8250 && CPUMIsGuestInRealModeEx(pMixedCtx))
8251 {
8252 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8253 }
8254
8255 /*
8256 * Load the guest-state into the VMCS.
8257 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8258 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8259 */
8260 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8261 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8262
8263 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8264 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8265 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8266
8267 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8268 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8269 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8270
8271 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8272 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8273
8274 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8275 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8276
8277 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8278 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8279 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8280
8281 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8282 determine we don't have to swap EFER after all. */
8283 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8284 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8285
8286 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8287 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8288
8289 /*
8290 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8291 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8292 */
8293 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8294 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8295
8296 /* Clear any unused and reserved bits. */
8297 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8298
8299 VMMRZCallRing3Enable(pVCpu);
8300
8301 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8302 return rc;
8303}
8304
8305
8306/**
8307 * Loads the state shared between the host and guest into the VMCS.
8308 *
8309 * @param pVM Pointer to the VM.
8310 * @param pVCpu Pointer to the VMCPU.
8311 * @param pCtx Pointer to the guest-CPU context.
8312 *
8313 * @remarks No-long-jump zone!!!
8314 */
8315static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8316{
8317 NOREF(pVM);
8318
8319 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8320 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8321
8322 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8323 {
8324 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8325 AssertRC(rc);
8326 }
8327
8328 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8329 {
8330 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8331 AssertRC(rc);
8332
8333 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8334 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8335 {
8336 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8337 AssertRC(rc);
8338 }
8339 }
8340
8341 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8342 {
8343#if HC_ARCH_BITS == 64
8344 if (pVM->hm.s.fAllow64BitGuests)
8345 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8346#endif
8347 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8348 }
8349
8350 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8351 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8352}
8353
8354
8355/**
8356 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8357 *
8358 * @param pVM Pointer to the VM.
8359 * @param pVCpu Pointer to the VMCPU.
8360 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8361 * out-of-sync. Make sure to update the required fields
8362 * before using them.
8363 */
8364DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8365{
8366 HMVMX_ASSERT_PREEMPT_SAFE();
8367
8368 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8369#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8370 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8371#endif
8372
8373 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8374 {
8375 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8376 AssertRC(rc);
8377 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8378 }
8379 else if (HMCPU_CF_VALUE(pVCpu))
8380 {
8381 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8382 AssertRC(rc);
8383 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8384 }
8385
8386 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8387 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8388 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8389 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8390}
8391
8392
8393/**
8394 * Does the preparations before executing guest code in VT-x.
8395 *
8396 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8397 * recompiler/IEM. We must be cautious what we do here regarding committing
8398 * guest-state information into the VMCS assuming we assuredly execute the
8399 * guest in VT-x mode.
8400 *
8401 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8402 * the common-state (TRPM/forceflags), we must undo those changes so that the
8403 * recompiler/IEM can (and should) use them when it resumes guest execution.
8404 * Otherwise such operations must be done when we can no longer exit to ring-3.
8405 *
8406 * @returns Strict VBox status code.
8407 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8408 * have been disabled.
8409 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8410 * double-fault into the guest.
8411 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8412 * dispatched directly.
8413 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8414 *
8415 * @param pVM Pointer to the VM.
8416 * @param pVCpu Pointer to the VMCPU.
8417 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8418 * out-of-sync. Make sure to update the required fields
8419 * before using them.
8420 * @param pVmxTransient Pointer to the VMX transient structure.
8421 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8422 * us ignore some of the reasons for returning to
8423 * ring-3, and return VINF_EM_DBG_STEPPED if event
8424 * dispatching took place.
8425 */
8426static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8427{
8428 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8429
8430#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8431 PGMRZDynMapFlushAutoSet(pVCpu);
8432#endif
8433
8434 /* Check force flag actions that might require us to go back to ring-3. */
8435 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8436 if (rc != VINF_SUCCESS)
8437 return rc;
8438
8439#ifndef IEM_VERIFICATION_MODE_FULL
8440 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8441 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8442 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8443 {
8444 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8445 RTGCPHYS GCPhysApicBase;
8446 GCPhysApicBase = pMixedCtx->msrApicBase;
8447 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8448
8449 /* Unalias any existing mapping. */
8450 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8451 AssertRCReturn(rc, rc);
8452
8453 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8454 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8455 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8456 AssertRCReturn(rc, rc);
8457
8458 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8459 }
8460#endif /* !IEM_VERIFICATION_MODE_FULL */
8461
8462 if (TRPMHasTrap(pVCpu))
8463 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8464 else if (!pVCpu->hm.s.Event.fPending)
8465 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8466
8467 /*
8468 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8469 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8470 */
8471 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8472 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8473 {
8474 Assert(rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
8475 return rc;
8476 }
8477
8478 /*
8479 * Load the guest state bits, we can handle longjmps/getting preempted here.
8480 *
8481 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8482 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8483 * Hence, this needs to be done -after- injection of events.
8484 */
8485 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8486
8487 /*
8488 * No longjmps to ring-3 from this point on!!!
8489 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8490 * This also disables flushing of the R0-logger instance (if any).
8491 */
8492 VMMRZCallRing3Disable(pVCpu);
8493
8494 /*
8495 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8496 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8497 *
8498 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8499 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8500 *
8501 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8502 * executing guest code.
8503 */
8504 pVmxTransient->uEflags = ASMIntDisableFlags();
8505 if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8506 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8507 && ( !fStepping /* Optimized for the non-stepping case, of course. */
8508 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8509 {
8510 hmR0VmxClearEventVmcs(pVCpu);
8511 ASMSetFlags(pVmxTransient->uEflags);
8512 VMMRZCallRing3Enable(pVCpu);
8513 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8514 return VINF_EM_RAW_TO_R3;
8515 }
8516
8517 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8518 {
8519 hmR0VmxClearEventVmcs(pVCpu);
8520 ASMSetFlags(pVmxTransient->uEflags);
8521 VMMRZCallRing3Enable(pVCpu);
8522 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8523 return VINF_EM_RAW_INTERRUPT;
8524 }
8525
8526 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8527 pVCpu->hm.s.Event.fPending = false;
8528
8529 return VINF_SUCCESS;
8530}
8531
8532
8533/**
8534 * Prepares to run guest code in VT-x and we've committed to doing so. This
8535 * means there is no backing out to ring-3 or anywhere else at this
8536 * point.
8537 *
8538 * @param pVM Pointer to the VM.
8539 * @param pVCpu Pointer to the VMCPU.
8540 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8541 * out-of-sync. Make sure to update the required fields
8542 * before using them.
8543 * @param pVmxTransient Pointer to the VMX transient structure.
8544 *
8545 * @remarks Called with preemption disabled.
8546 * @remarks No-long-jump zone!!!
8547 */
8548static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8549{
8550 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8551 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8552 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8553
8554 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8555 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8556
8557#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8558 if (!CPUMIsGuestFPUStateActive(pVCpu))
8559 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8560 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8561#endif
8562
8563 if ( pVCpu->hm.s.fUseGuestFpu
8564 && !CPUMIsGuestFPUStateActive(pVCpu))
8565 {
8566 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8567 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8568 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8569 }
8570
8571 /*
8572 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8573 */
8574 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8575 && pVCpu->hm.s.vmx.cMsrs > 0)
8576 {
8577 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8578 }
8579
8580 /*
8581 * Load the host state bits as we may've been preempted (only happens when
8582 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8583 */
8584 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8585 {
8586 /* This ASSUMES that pfnStartVM has been set up already. */
8587 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8588 AssertRC(rc);
8589 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8590 }
8591 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8592
8593 /*
8594 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8595 */
8596 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8597 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8598 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8599
8600 /* Store status of the shared guest-host state at the time of VM-entry. */
8601#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8602 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8603 {
8604 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8605 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8606 }
8607 else
8608#endif
8609 {
8610 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8611 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8612 }
8613 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8614
8615 /*
8616 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8617 */
8618 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8619 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8620
8621 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8622 RTCPUID idCurrentCpu = pCpu->idCpu;
8623 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8624 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8625 {
8626 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8627 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8628 }
8629
8630 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8631 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8632 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8633 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8634
8635 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8636
8637 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8638 to start executing. */
8639
8640 /*
8641 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8642 */
8643 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8644 {
8645 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8646 {
8647 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8648 AssertRC(rc2);
8649 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8650 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8651 true /* fUpdateHostMsr */);
8652 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8653 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8654 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8655 }
8656 else
8657 {
8658 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8659 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8660 }
8661 }
8662
8663#ifdef VBOX_STRICT
8664 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8665 hmR0VmxCheckHostEferMsr(pVCpu);
8666 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8667#endif
8668#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8669 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8670 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8671 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8672#endif
8673}
8674
8675
8676/**
8677 * Performs some essential restoration of state after running guest code in
8678 * VT-x.
8679 *
8680 * @param pVM Pointer to the VM.
8681 * @param pVCpu Pointer to the VMCPU.
8682 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8683 * out-of-sync. Make sure to update the required fields
8684 * before using them.
8685 * @param pVmxTransient Pointer to the VMX transient structure.
8686 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8687 *
8688 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8689 *
8690 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8691 * unconditionally when it is safe to do so.
8692 */
8693static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8694{
8695 NOREF(pVM);
8696
8697 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8698
8699 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8700 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8701 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8702 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8703 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8704 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8705
8706 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8707 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset); /** @todo use SUPReadTSC() eventually. */
8708
8709 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8710 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8711 Assert(!(ASMGetFlags() & X86_EFL_IF));
8712 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8713
8714#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8715 if (CPUMIsGuestFPUStateActive(pVCpu))
8716 {
8717 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8718 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8719 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8720 }
8721#endif
8722
8723#if HC_ARCH_BITS == 64
8724 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8725#endif
8726 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8727#ifdef VBOX_STRICT
8728 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8729#endif
8730 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8731 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8732
8733 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8734 uint32_t uExitReason;
8735 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8736 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8737 AssertRC(rc);
8738 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8739 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8740
8741 /* Update the VM-exit history array. */
8742 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8743
8744 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8745 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8746 {
8747 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8748 pVmxTransient->fVMEntryFailed));
8749 return;
8750 }
8751
8752 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8753 {
8754 /** @todo We can optimize this by only syncing with our force-flags when
8755 * really needed and keeping the VMCS state as it is for most
8756 * VM-exits. */
8757 /* Update the guest interruptibility-state from the VMCS. */
8758 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8759
8760#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8761 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8762 AssertRC(rc);
8763#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8764 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8765 AssertRC(rc);
8766#endif
8767
8768 /*
8769 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8770 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8771 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8772 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8773 */
8774 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8775 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8776 {
8777 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8778 AssertRC(rc);
8779 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8780 }
8781 }
8782}
8783
8784
8785/**
8786 * Runs the guest code using VT-x the normal way.
8787 *
8788 * @returns VBox status code.
8789 * @param pVM Pointer to the VM.
8790 * @param pVCpu Pointer to the VMCPU.
8791 * @param pCtx Pointer to the guest-CPU context.
8792 *
8793 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8794 */
8795static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8796{
8797 VMXTRANSIENT VmxTransient;
8798 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8799 int rc = VERR_INTERNAL_ERROR_5;
8800 uint32_t cLoops = 0;
8801
8802 for (;; cLoops++)
8803 {
8804 Assert(!HMR0SuspendPending());
8805 HMVMX_ASSERT_CPU_SAFE();
8806
8807 /* Preparatory work for running guest code, this may force us to return
8808 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8809 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8810 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8811 if (rc != VINF_SUCCESS)
8812 break;
8813
8814 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8815 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8816 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8817
8818 /* Restore any residual host-state and save any bits shared between host
8819 and guest into the guest-CPU state. Re-enables interrupts! */
8820 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8821
8822 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8823 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8824 {
8825 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8826 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8827 return rc;
8828 }
8829
8830 /* Profile the VM-exit. */
8831 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8832 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8833 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8834 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8835 HMVMX_START_EXIT_DISPATCH_PROF();
8836
8837 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8838 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8839 {
8840 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8841 hmR0VmxSaveGuestState(pVCpu, pCtx);
8842 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8843 }
8844
8845 /* Handle the VM-exit. */
8846#ifdef HMVMX_USE_FUNCTION_TABLE
8847 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8848#else
8849 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8850#endif
8851 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8852 if (rc != VINF_SUCCESS)
8853 break;
8854 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8855 {
8856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8857 rc = VINF_EM_RAW_INTERRUPT;
8858 break;
8859 }
8860 }
8861
8862 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8863 return rc;
8864}
8865
8866
8867/**
8868 * Single steps guest code using VT-x.
8869 *
8870 * @returns VBox status code.
8871 * @param pVM Pointer to the VM.
8872 * @param pVCpu Pointer to the VMCPU.
8873 * @param pCtx Pointer to the guest-CPU context.
8874 *
8875 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8876 */
8877static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8878{
8879 VMXTRANSIENT VmxTransient;
8880 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8881 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8882 uint32_t cLoops = 0;
8883 uint16_t uCsStart = pCtx->cs.Sel;
8884 uint64_t uRipStart = pCtx->rip;
8885
8886 for (;; cLoops++)
8887 {
8888 Assert(!HMR0SuspendPending());
8889 HMVMX_ASSERT_CPU_SAFE();
8890
8891 /* Preparatory work for running guest code, this may force us to return
8892 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8893 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8894 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8895 if (rcStrict != VINF_SUCCESS)
8896 break;
8897
8898 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8899 rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8900 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8901
8902 /* Restore any residual host-state and save any bits shared between host
8903 and guest into the guest-CPU state. Re-enables interrupts! */
8904 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8905
8906 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8907 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
8908 {
8909 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8910 hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
8911 return VBOXSTRICTRC_TODO(rcStrict);
8912 }
8913
8914 /* Profile the VM-exit. */
8915 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8917 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8918 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8919 HMVMX_START_EXIT_DISPATCH_PROF();
8920
8921 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8922 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8923 {
8924 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8925 hmR0VmxSaveGuestState(pVCpu, pCtx);
8926 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8927 }
8928
8929 /* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8930 rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8931 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8932 if (rcStrict != VINF_SUCCESS)
8933 break;
8934 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8935 {
8936 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8937 rcStrict = VINF_EM_RAW_INTERRUPT;
8938 break;
8939 }
8940
8941 /*
8942 * Did the RIP change, if so, consider it a single step.
8943 * Otherwise, make sure one of the TFs gets set.
8944 */
8945 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8946 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8947 AssertRCReturn(rc2, rc2);
8948 if ( pCtx->rip != uRipStart
8949 || pCtx->cs.Sel != uCsStart)
8950 {
8951 rcStrict = VINF_EM_DBG_STEPPED;
8952 break;
8953 }
8954 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8955 }
8956
8957 /*
8958 * Clear the X86_EFL_TF if necessary.
8959 */
8960 if (pVCpu->hm.s.fClearTrapFlag)
8961 {
8962 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8963 AssertRCReturn(rc2, rc2);
8964 pVCpu->hm.s.fClearTrapFlag = false;
8965 pCtx->eflags.Bits.u1TF = 0;
8966 }
8967 /** @todo there seems to be issues with the resume flag when the monitor trap
8968 * flag is pending without being used. Seen early in bios init when
8969 * accessing APIC page in protected mode. */
8970
8971 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8972 return VBOXSTRICTRC_TODO(rcStrict);
8973}
8974
8975
8976/**
8977 * Runs the guest code using VT-x.
8978 *
8979 * @returns VBox status code.
8980 * @param pVM Pointer to the VM.
8981 * @param pVCpu Pointer to the VMCPU.
8982 * @param pCtx Pointer to the guest-CPU context.
8983 */
8984VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8985{
8986 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8987 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8988 HMVMX_ASSERT_PREEMPT_SAFE();
8989
8990 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8991
8992 int rc;
8993 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8994 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8995 else
8996 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8997
8998 if (rc == VERR_EM_INTERPRETER)
8999 rc = VINF_EM_RAW_EMULATE_INSTR;
9000 else if (rc == VINF_EM_RESET)
9001 rc = VINF_EM_TRIPLE_FAULT;
9002
9003 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
9004 if (RT_FAILURE(rc2))
9005 {
9006 pVCpu->hm.s.u32HMError = rc;
9007 rc = rc2;
9008 }
9009 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
9010 return rc;
9011}
9012
9013
9014#ifndef HMVMX_USE_FUNCTION_TABLE
9015DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9016{
9017#ifdef DEBUG_ramshankar
9018# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9019# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9020#endif
9021 int rc;
9022 switch (rcReason)
9023 {
9024 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9025 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9026 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9027 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9028 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9029 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9030 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9031 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9032 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9033 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9034 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9035 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9036 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9037 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9038 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9039 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9040 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9041 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9042 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9043 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9044 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9045 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9046 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9047 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9048 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9049 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9050 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9051 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9052 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9053 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9054 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9055 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9056 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9057 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9058
9059 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9060 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9061 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9062 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9063 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9064 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9065 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9066 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9067 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9068
9069 case VMX_EXIT_VMCLEAR:
9070 case VMX_EXIT_VMLAUNCH:
9071 case VMX_EXIT_VMPTRLD:
9072 case VMX_EXIT_VMPTRST:
9073 case VMX_EXIT_VMREAD:
9074 case VMX_EXIT_VMRESUME:
9075 case VMX_EXIT_VMWRITE:
9076 case VMX_EXIT_VMXOFF:
9077 case VMX_EXIT_VMXON:
9078 case VMX_EXIT_INVEPT:
9079 case VMX_EXIT_INVVPID:
9080 case VMX_EXIT_VMFUNC:
9081 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9082 break;
9083 default:
9084 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9085 break;
9086 }
9087 return rc;
9088}
9089#endif /* !HMVMX_USE_FUNCTION_TABLE */
9090
9091
9092/**
9093 * Single-stepping VM-exit filtering.
9094 *
9095 * This is preprocessing the exits and deciding whether we've gotten far enough
9096 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9097 * performed.
9098 *
9099 * @returns Strict VBox status code.
9100 * @param pVCpu The virtual CPU of the calling EMT.
9101 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9102 * out-of-sync. Make sure to update the required
9103 * fields before using them.
9104 * @param pVmxTransient Pointer to the VMX-transient structure.
9105 * @param uExitReason The VM-exit reason.
9106 */
9107DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9108 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9109{
9110 switch (uExitReason)
9111 {
9112 case VMX_EXIT_XCPT_OR_NMI:
9113 {
9114 /* Check for host NMI. */
9115 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9116 AssertRCReturn(rc2, rc2);
9117 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9118 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9119 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9120 /* fall thru */
9121 }
9122
9123 case VMX_EXIT_EPT_MISCONFIG:
9124 case VMX_EXIT_TRIPLE_FAULT:
9125 case VMX_EXIT_APIC_ACCESS:
9126 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9127 case VMX_EXIT_TASK_SWITCH:
9128
9129 /* Instruction specific VM-exits: */
9130 case VMX_EXIT_IO_INSTR:
9131 case VMX_EXIT_CPUID:
9132 case VMX_EXIT_RDTSC:
9133 case VMX_EXIT_RDTSCP:
9134 case VMX_EXIT_MOV_CRX:
9135 case VMX_EXIT_MWAIT:
9136 case VMX_EXIT_MONITOR:
9137 case VMX_EXIT_RDMSR:
9138 case VMX_EXIT_WRMSR:
9139 case VMX_EXIT_MOV_DRX:
9140 case VMX_EXIT_HLT:
9141 case VMX_EXIT_INVD:
9142 case VMX_EXIT_INVLPG:
9143 case VMX_EXIT_RSM:
9144 case VMX_EXIT_PAUSE:
9145 case VMX_EXIT_XDTR_ACCESS:
9146 case VMX_EXIT_TR_ACCESS:
9147 case VMX_EXIT_WBINVD:
9148 case VMX_EXIT_XSETBV:
9149 case VMX_EXIT_RDRAND:
9150 case VMX_EXIT_INVPCID:
9151 case VMX_EXIT_GETSEC:
9152 case VMX_EXIT_RDPMC:
9153 case VMX_EXIT_VMCALL:
9154 case VMX_EXIT_VMCLEAR:
9155 case VMX_EXIT_VMLAUNCH:
9156 case VMX_EXIT_VMPTRLD:
9157 case VMX_EXIT_VMPTRST:
9158 case VMX_EXIT_VMREAD:
9159 case VMX_EXIT_VMRESUME:
9160 case VMX_EXIT_VMWRITE:
9161 case VMX_EXIT_VMXOFF:
9162 case VMX_EXIT_VMXON:
9163 case VMX_EXIT_INVEPT:
9164 case VMX_EXIT_INVVPID:
9165 case VMX_EXIT_VMFUNC:
9166 {
9167 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9168 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9169 AssertRCReturn(rc2, rc2);
9170 if ( pMixedCtx->rip != uRipStart
9171 || pMixedCtx->cs.Sel != uCsStart)
9172 return VINF_EM_DBG_STEPPED;
9173 break;
9174 }
9175 }
9176
9177 /*
9178 * Normal processing.
9179 */
9180#ifdef HMVMX_USE_FUNCTION_TABLE
9181 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9182#else
9183 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9184#endif
9185}
9186
9187
9188#ifdef DEBUG
9189/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9190# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9191 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9192
9193# define HMVMX_ASSERT_PREEMPT_CPUID() \
9194 do \
9195 { \
9196 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9197 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9198 } while (0)
9199
9200# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9201 do { \
9202 AssertPtr(pVCpu); \
9203 AssertPtr(pMixedCtx); \
9204 AssertPtr(pVmxTransient); \
9205 Assert(pVmxTransient->fVMEntryFailed == false); \
9206 Assert(ASMIntAreEnabled()); \
9207 HMVMX_ASSERT_PREEMPT_SAFE(); \
9208 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9209 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9210 HMVMX_ASSERT_PREEMPT_SAFE(); \
9211 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9212 HMVMX_ASSERT_PREEMPT_CPUID(); \
9213 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9214 } while (0)
9215
9216# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9217 do { \
9218 Log4Func(("\n")); \
9219 } while (0)
9220#else /* Release builds */
9221# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9222 do { \
9223 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9224 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9225 } while (0)
9226# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9227#endif
9228
9229
9230/**
9231 * Advances the guest RIP after reading it from the VMCS.
9232 *
9233 * @returns VBox status code.
9234 * @param pVCpu Pointer to the VMCPU.
9235 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9236 * out-of-sync. Make sure to update the required fields
9237 * before using them.
9238 * @param pVmxTransient Pointer to the VMX transient structure.
9239 *
9240 * @remarks No-long-jump zone!!!
9241 */
9242DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9243{
9244 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9245 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9246 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9247 AssertRCReturn(rc, rc);
9248
9249 pMixedCtx->rip += pVmxTransient->cbInstr;
9250 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9251
9252 /*
9253 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9254 * pending debug exception field as it takes care of priority of events.
9255 *
9256 * See Intel spec. 32.2.1 "Debug Exceptions".
9257 */
9258 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9259
9260 return rc;
9261}
9262
9263
9264/**
9265 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9266 * and update error record fields accordingly.
9267 *
9268 * @return VMX_IGS_* return codes.
9269 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9270 * wrong with the guest state.
9271 *
9272 * @param pVM Pointer to the VM.
9273 * @param pVCpu Pointer to the VMCPU.
9274 * @param pCtx Pointer to the guest-CPU state.
9275 *
9276 * @remarks This function assumes our cache of the VMCS controls
9277 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9278 */
9279static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9280{
9281#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9282#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9283 uError = (err); \
9284 break; \
9285 } else do { } while (0)
9286
9287 int rc;
9288 uint32_t uError = VMX_IGS_ERROR;
9289 uint32_t u32Val;
9290 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9291
9292 do
9293 {
9294 /*
9295 * CR0.
9296 */
9297 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9298 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9299 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9300 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9301 if (fUnrestrictedGuest)
9302 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9303
9304 uint32_t u32GuestCR0;
9305 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9306 AssertRCBreak(rc);
9307 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9308 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9309 if ( !fUnrestrictedGuest
9310 && (u32GuestCR0 & X86_CR0_PG)
9311 && !(u32GuestCR0 & X86_CR0_PE))
9312 {
9313 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9314 }
9315
9316 /*
9317 * CR4.
9318 */
9319 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9320 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9321
9322 uint32_t u32GuestCR4;
9323 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9324 AssertRCBreak(rc);
9325 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9326 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9327
9328 /*
9329 * IA32_DEBUGCTL MSR.
9330 */
9331 uint64_t u64Val;
9332 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9333 AssertRCBreak(rc);
9334 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9335 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9336 {
9337 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9338 }
9339 uint64_t u64DebugCtlMsr = u64Val;
9340
9341#ifdef VBOX_STRICT
9342 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9343 AssertRCBreak(rc);
9344 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9345#endif
9346 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9347
9348 /*
9349 * RIP and RFLAGS.
9350 */
9351 uint32_t u32Eflags;
9352#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9353 if (HMVMX_IS_64BIT_HOST_MODE())
9354 {
9355 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9356 AssertRCBreak(rc);
9357 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9358 if ( !fLongModeGuest
9359 || !pCtx->cs.Attr.n.u1Long)
9360 {
9361 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9362 }
9363 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9364 * must be identical if the "IA-32e mode guest" VM-entry
9365 * control is 1 and CS.L is 1. No check applies if the
9366 * CPU supports 64 linear-address bits. */
9367
9368 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9369 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9370 AssertRCBreak(rc);
9371 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9372 VMX_IGS_RFLAGS_RESERVED);
9373 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9374 u32Eflags = u64Val;
9375 }
9376 else
9377#endif
9378 {
9379 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9380 AssertRCBreak(rc);
9381 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9382 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9383 }
9384
9385 if ( fLongModeGuest
9386 || ( fUnrestrictedGuest
9387 && !(u32GuestCR0 & X86_CR0_PE)))
9388 {
9389 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9390 }
9391
9392 uint32_t u32EntryInfo;
9393 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9394 AssertRCBreak(rc);
9395 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9396 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9397 {
9398 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9399 }
9400
9401 /*
9402 * 64-bit checks.
9403 */
9404#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9405 if (HMVMX_IS_64BIT_HOST_MODE())
9406 {
9407 if ( fLongModeGuest
9408 && !fUnrestrictedGuest)
9409 {
9410 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9411 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9412 }
9413
9414 if ( !fLongModeGuest
9415 && (u32GuestCR4 & X86_CR4_PCIDE))
9416 {
9417 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9418 }
9419
9420 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9421 * 51:32 beyond the processor's physical-address width are 0. */
9422
9423 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9424 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9425 {
9426 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9427 }
9428
9429 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9430 AssertRCBreak(rc);
9431 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9432
9433 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9434 AssertRCBreak(rc);
9435 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9436 }
9437#endif
9438
9439 /*
9440 * PERF_GLOBAL MSR.
9441 */
9442 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9443 {
9444 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9445 AssertRCBreak(rc);
9446 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9447 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9448 }
9449
9450 /*
9451 * PAT MSR.
9452 */
9453 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9454 {
9455 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9456 AssertRCBreak(rc);
9457 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9458 for (unsigned i = 0; i < 8; i++)
9459 {
9460 uint8_t u8Val = (u64Val & 0xff);
9461 if ( u8Val != 0 /* UC */
9462 && u8Val != 1 /* WC */
9463 && u8Val != 4 /* WT */
9464 && u8Val != 5 /* WP */
9465 && u8Val != 6 /* WB */
9466 && u8Val != 7 /* UC- */)
9467 {
9468 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9469 }
9470 u64Val >>= 8;
9471 }
9472 }
9473
9474 /*
9475 * EFER MSR.
9476 */
9477 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9478 {
9479 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9480 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9481 AssertRCBreak(rc);
9482 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9483 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9484 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9485 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9486 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9487 || !(u32GuestCR0 & X86_CR0_PG)
9488 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9489 VMX_IGS_EFER_LMA_LME_MISMATCH);
9490 }
9491
9492 /*
9493 * Segment registers.
9494 */
9495 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9496 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9497 if (!(u32Eflags & X86_EFL_VM))
9498 {
9499 /* CS */
9500 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9501 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9502 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9503 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9504 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9505 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9506 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9507 /* CS cannot be loaded with NULL in protected mode. */
9508 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9509 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9510 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9511 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9512 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9513 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9514 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9515 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9516 else
9517 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9518
9519 /* SS */
9520 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9521 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9522 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9523 if ( !(pCtx->cr0 & X86_CR0_PE)
9524 || pCtx->cs.Attr.n.u4Type == 3)
9525 {
9526 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9527 }
9528 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9529 {
9530 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9531 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9532 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9533 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9534 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9535 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9536 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9537 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9538 }
9539
9540 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9541 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9542 {
9543 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9544 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9545 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9546 || pCtx->ds.Attr.n.u4Type > 11
9547 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9548 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9549 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9550 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9551 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9552 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9553 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9554 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9555 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9556 }
9557 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9558 {
9559 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9560 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9561 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9562 || pCtx->es.Attr.n.u4Type > 11
9563 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9564 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9565 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9566 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9567 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9568 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9569 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9570 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9571 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9572 }
9573 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9574 {
9575 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9576 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9577 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9578 || pCtx->fs.Attr.n.u4Type > 11
9579 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9580 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9581 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9582 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9583 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9584 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9585 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9586 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9587 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9588 }
9589 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9590 {
9591 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9592 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9593 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9594 || pCtx->gs.Attr.n.u4Type > 11
9595 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9596 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9597 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9598 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9599 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9600 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9601 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9602 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9603 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9604 }
9605 /* 64-bit capable CPUs. */
9606#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9607 if (HMVMX_IS_64BIT_HOST_MODE())
9608 {
9609 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9610 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9611 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9612 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9613 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9614 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9615 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9616 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9617 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9618 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9619 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9620 }
9621#endif
9622 }
9623 else
9624 {
9625 /* V86 mode checks. */
9626 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9627 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9628 {
9629 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9630 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9631 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9632 }
9633 else
9634 {
9635 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9636 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9637 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9638 }
9639
9640 /* CS */
9641 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9642 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9643 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9644 /* SS */
9645 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9646 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9647 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9648 /* DS */
9649 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9650 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9651 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9652 /* ES */
9653 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9654 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9655 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9656 /* FS */
9657 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9658 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9659 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9660 /* GS */
9661 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9662 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9663 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9664 /* 64-bit capable CPUs. */
9665#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9666 if (HMVMX_IS_64BIT_HOST_MODE())
9667 {
9668 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9669 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9670 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9671 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9672 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9673 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9674 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9675 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9676 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9677 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9678 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9679 }
9680#endif
9681 }
9682
9683 /*
9684 * TR.
9685 */
9686 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9687 /* 64-bit capable CPUs. */
9688#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9689 if (HMVMX_IS_64BIT_HOST_MODE())
9690 {
9691 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9692 }
9693#endif
9694 if (fLongModeGuest)
9695 {
9696 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9697 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9698 }
9699 else
9700 {
9701 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9702 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9703 VMX_IGS_TR_ATTR_TYPE_INVALID);
9704 }
9705 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9706 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9707 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9708 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9709 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9710 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9711 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9712 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9713
9714 /*
9715 * GDTR and IDTR.
9716 */
9717#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9718 if (HMVMX_IS_64BIT_HOST_MODE())
9719 {
9720 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9721 AssertRCBreak(rc);
9722 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9723
9724 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9725 AssertRCBreak(rc);
9726 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9727 }
9728#endif
9729
9730 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9731 AssertRCBreak(rc);
9732 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9733
9734 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9735 AssertRCBreak(rc);
9736 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9737
9738 /*
9739 * Guest Non-Register State.
9740 */
9741 /* Activity State. */
9742 uint32_t u32ActivityState;
9743 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9744 AssertRCBreak(rc);
9745 HMVMX_CHECK_BREAK( !u32ActivityState
9746 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9747 VMX_IGS_ACTIVITY_STATE_INVALID);
9748 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9749 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9750 uint32_t u32IntrState;
9751 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9752 AssertRCBreak(rc);
9753 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9754 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9755 {
9756 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9757 }
9758
9759 /** @todo Activity state and injecting interrupts. Left as a todo since we
9760 * currently don't use activity states but ACTIVE. */
9761
9762 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9763 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9764
9765 /* Guest interruptibility-state. */
9766 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9767 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9768 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9769 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9770 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9771 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9772 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9773 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9774 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9775 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9776 {
9777 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9778 {
9779 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9780 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9781 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9782 }
9783 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9784 {
9785 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9786 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9787 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9788 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9789 }
9790 }
9791 /** @todo Assumes the processor is not in SMM. */
9792 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9793 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9794 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9795 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9796 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9797 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9798 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9799 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9800 {
9801 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9802 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9803 }
9804
9805 /* Pending debug exceptions. */
9806 if (HMVMX_IS_64BIT_HOST_MODE())
9807 {
9808 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9809 AssertRCBreak(rc);
9810 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9811 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9812 u32Val = u64Val; /* For pending debug exceptions checks below. */
9813 }
9814 else
9815 {
9816 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9817 AssertRCBreak(rc);
9818 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9819 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9820 }
9821
9822 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9823 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9824 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9825 {
9826 if ( (u32Eflags & X86_EFL_TF)
9827 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9828 {
9829 /* Bit 14 is PendingDebug.BS. */
9830 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9831 }
9832 if ( !(u32Eflags & X86_EFL_TF)
9833 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9834 {
9835 /* Bit 14 is PendingDebug.BS. */
9836 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9837 }
9838 }
9839
9840 /* VMCS link pointer. */
9841 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9842 AssertRCBreak(rc);
9843 if (u64Val != UINT64_C(0xffffffffffffffff))
9844 {
9845 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9846 /** @todo Bits beyond the processor's physical-address width MBZ. */
9847 /** @todo 32-bit located in memory referenced by value of this field (as a
9848 * physical address) must contain the processor's VMCS revision ID. */
9849 /** @todo SMM checks. */
9850 }
9851
9852 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9853 * not using Nested Paging? */
9854 if ( pVM->hm.s.fNestedPaging
9855 && !fLongModeGuest
9856 && CPUMIsGuestInPAEModeEx(pCtx))
9857 {
9858 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9859 AssertRCBreak(rc);
9860 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9861
9862 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9863 AssertRCBreak(rc);
9864 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9865
9866 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9867 AssertRCBreak(rc);
9868 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9869
9870 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9871 AssertRCBreak(rc);
9872 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9873 }
9874
9875 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9876 if (uError == VMX_IGS_ERROR)
9877 uError = VMX_IGS_REASON_NOT_FOUND;
9878 } while (0);
9879
9880 pVCpu->hm.s.u32HMError = uError;
9881 return uError;
9882
9883#undef HMVMX_ERROR_BREAK
9884#undef HMVMX_CHECK_BREAK
9885}
9886
9887/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9888/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9889/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9890
9891/** @name VM-exit handlers.
9892 * @{
9893 */
9894
9895/**
9896 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9897 */
9898HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9899{
9900 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9901 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9902 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9903 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9904 return VINF_SUCCESS;
9905 return VINF_EM_RAW_INTERRUPT;
9906}
9907
9908
9909/**
9910 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9911 */
9912HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9913{
9914 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9915 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9916
9917 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9918 AssertRCReturn(rc, rc);
9919
9920 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9921 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9922 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9923 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9924
9925 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9926 {
9927 /*
9928 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9929 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9930 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9931 *
9932 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9933 */
9934 VMXDispatchHostNmi();
9935 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9936 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9937 return VINF_SUCCESS;
9938 }
9939
9940 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9941 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9942 if (RT_UNLIKELY(rc != VINF_SUCCESS))
9943 {
9944 if (rc == VINF_HM_DOUBLE_FAULT)
9945 rc = VINF_SUCCESS;
9946 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9947 return rc;
9948 }
9949
9950 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9951 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9952 switch (uIntType)
9953 {
9954 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9955 Assert(uVector == X86_XCPT_DB);
9956 /* no break */
9957 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9958 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9959 /* no break */
9960 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9961 {
9962 switch (uVector)
9963 {
9964 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9965 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9966 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9967 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9968 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9969 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9970#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9971 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9972 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9973 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9974 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9975 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9976 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9977 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9978 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9979 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9980 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9981 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9982 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9983#endif
9984 default:
9985 {
9986 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9987 AssertRCReturn(rc, rc);
9988
9989 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9990 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9991 {
9992 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9993 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9994 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9995
9996 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9997 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9998 AssertRCReturn(rc, rc);
9999 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10000 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10001 0 /* GCPtrFaultAddress */);
10002 AssertRCReturn(rc, rc);
10003 }
10004 else
10005 {
10006 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10007 pVCpu->hm.s.u32HMError = uVector;
10008 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10009 }
10010 break;
10011 }
10012 }
10013 break;
10014 }
10015
10016 default:
10017 {
10018 pVCpu->hm.s.u32HMError = uExitIntInfo;
10019 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10020 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10021 break;
10022 }
10023 }
10024 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10025 return rc;
10026}
10027
10028
10029/**
10030 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10031 */
10032HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10033{
10034 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10035
10036 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
10037 hmR0VmxClearIntWindowExitVmcs(pVCpu);
10038
10039 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10040 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
10041 return VINF_SUCCESS;
10042}
10043
10044
10045/**
10046 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
10047 */
10048HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10049{
10050 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10051 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
10052 {
10053 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
10054 HMVMX_RETURN_UNEXPECTED_EXIT();
10055 }
10056
10057 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
10058
10059 /*
10060 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
10061 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
10062 */
10063 uint32_t uIntrState = 0;
10064 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10065 AssertRCReturn(rc, rc);
10066
10067 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
10068 if ( fBlockSti
10069 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
10070 {
10071 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10072 }
10073
10074 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
10075 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
10076
10077 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10078 return VINF_SUCCESS;
10079}
10080
10081
10082/**
10083 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10084 */
10085HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10086{
10087 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10088 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10089 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10090}
10091
10092
10093/**
10094 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10095 */
10096HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10097{
10098 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10099 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10100 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10101}
10102
10103
10104/**
10105 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10106 */
10107HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10108{
10109 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10110 PVM pVM = pVCpu->CTX_SUFF(pVM);
10111 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10112 if (RT_LIKELY(rc == VINF_SUCCESS))
10113 {
10114 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10115 Assert(pVmxTransient->cbInstr == 2);
10116 }
10117 else
10118 {
10119 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10120 rc = VERR_EM_INTERPRETER;
10121 }
10122 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10123 return rc;
10124}
10125
10126
10127/**
10128 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10129 */
10130HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10131{
10132 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10133 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10134 AssertRCReturn(rc, rc);
10135
10136 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10137 return VINF_EM_RAW_EMULATE_INSTR;
10138
10139 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10140 HMVMX_RETURN_UNEXPECTED_EXIT();
10141}
10142
10143
10144/**
10145 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10146 */
10147HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10148{
10149 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10150 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10151 AssertRCReturn(rc, rc);
10152
10153 PVM pVM = pVCpu->CTX_SUFF(pVM);
10154 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10155 if (RT_LIKELY(rc == VINF_SUCCESS))
10156 {
10157 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10158 Assert(pVmxTransient->cbInstr == 2);
10159 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10160 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10161 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10162 }
10163 else
10164 rc = VERR_EM_INTERPRETER;
10165 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10166 return rc;
10167}
10168
10169
10170/**
10171 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10172 */
10173HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10174{
10175 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10176 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10177 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10178 AssertRCReturn(rc, rc);
10179
10180 PVM pVM = pVCpu->CTX_SUFF(pVM);
10181 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10182 if (RT_LIKELY(rc == VINF_SUCCESS))
10183 {
10184 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10185 Assert(pVmxTransient->cbInstr == 3);
10186 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10187 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10188 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10189 }
10190 else
10191 {
10192 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10193 rc = VERR_EM_INTERPRETER;
10194 }
10195 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10196 return rc;
10197}
10198
10199
10200/**
10201 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10202 */
10203HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10204{
10205 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10206 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10207 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10208 AssertRCReturn(rc, rc);
10209
10210 PVM pVM = pVCpu->CTX_SUFF(pVM);
10211 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10212 if (RT_LIKELY(rc == VINF_SUCCESS))
10213 {
10214 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10215 Assert(pVmxTransient->cbInstr == 2);
10216 }
10217 else
10218 {
10219 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10220 rc = VERR_EM_INTERPRETER;
10221 }
10222 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10223 return rc;
10224}
10225
10226
10227/**
10228 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10229 */
10230HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10231{
10232 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10233
10234 int rc = VERR_NOT_SUPPORTED;
10235 if (GIMAreHypercallsEnabled(pVCpu))
10236 {
10237 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10238 AssertRCReturn(rc, rc);
10239
10240 rc = GIMHypercall(pVCpu, pMixedCtx);
10241 }
10242 if (rc != VINF_SUCCESS)
10243 {
10244 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10245 rc = VINF_SUCCESS;
10246 }
10247
10248 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10249 return rc;
10250}
10251
10252
10253/**
10254 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10255 */
10256HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10257{
10258 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10259 PVM pVM = pVCpu->CTX_SUFF(pVM);
10260 Assert(!pVM->hm.s.fNestedPaging);
10261
10262 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10263 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10264 AssertRCReturn(rc, rc);
10265
10266 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10267 rc = VBOXSTRICTRC_VAL(rc2);
10268 if (RT_LIKELY(rc == VINF_SUCCESS))
10269 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10270 else
10271 {
10272 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10273 pVmxTransient->uExitQualification, rc));
10274 }
10275 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10276 return rc;
10277}
10278
10279
10280/**
10281 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10282 */
10283HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10284{
10285 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10286 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10287 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10288 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10289 AssertRCReturn(rc, rc);
10290
10291 PVM pVM = pVCpu->CTX_SUFF(pVM);
10292 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10293 if (RT_LIKELY(rc == VINF_SUCCESS))
10294 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10295 else
10296 {
10297 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10298 rc = VERR_EM_INTERPRETER;
10299 }
10300 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10301 return rc;
10302}
10303
10304
10305/**
10306 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10307 */
10308HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10309{
10310 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10311 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10312 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10313 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10314 AssertRCReturn(rc, rc);
10315
10316 PVM pVM = pVCpu->CTX_SUFF(pVM);
10317 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10318 rc = VBOXSTRICTRC_VAL(rc2);
10319 if (RT_LIKELY( rc == VINF_SUCCESS
10320 || rc == VINF_EM_HALT))
10321 {
10322 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10323 AssertRCReturn(rc3, rc3);
10324
10325 if ( rc == VINF_EM_HALT
10326 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10327 {
10328 rc = VINF_SUCCESS;
10329 }
10330 }
10331 else
10332 {
10333 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10334 rc = VERR_EM_INTERPRETER;
10335 }
10336 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10337 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10338 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10339 return rc;
10340}
10341
10342
10343/**
10344 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10345 */
10346HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10347{
10348 /*
10349 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10350 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10351 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10352 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10353 */
10354 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10355 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10356 HMVMX_RETURN_UNEXPECTED_EXIT();
10357}
10358
10359
10360/**
10361 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10362 */
10363HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10364{
10365 /*
10366 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10367 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10368 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10369 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10370 */
10371 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10372 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10373 HMVMX_RETURN_UNEXPECTED_EXIT();
10374}
10375
10376
10377/**
10378 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10379 */
10380HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10381{
10382 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10383 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10384 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10385 HMVMX_RETURN_UNEXPECTED_EXIT();
10386}
10387
10388
10389/**
10390 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10391 */
10392HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10393{
10394 /*
10395 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10396 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10397 * See Intel spec. 25.3 "Other Causes of VM-exits".
10398 */
10399 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10400 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10401 HMVMX_RETURN_UNEXPECTED_EXIT();
10402}
10403
10404
10405/**
10406 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10407 * VM-exit.
10408 */
10409HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10410{
10411 /*
10412 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10413 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10414 *
10415 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10416 * See Intel spec. "23.8 Restrictions on VMX operation".
10417 */
10418 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10419 return VINF_SUCCESS;
10420}
10421
10422
10423/**
10424 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10425 * VM-exit.
10426 */
10427HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10428{
10429 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10430 return VINF_EM_RESET;
10431}
10432
10433
10434/**
10435 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10436 */
10437HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10438{
10439 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10440 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10441 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10442 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10443 AssertRCReturn(rc, rc);
10444
10445 pMixedCtx->rip++;
10446 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10447 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10448 rc = VINF_SUCCESS;
10449 else
10450 rc = VINF_EM_HALT;
10451
10452 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10453 if (rc != VINF_SUCCESS)
10454 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHltToR3);
10455 return rc;
10456}
10457
10458
10459/**
10460 * VM-exit handler for instructions that result in a #UD exception delivered to
10461 * the guest.
10462 */
10463HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10464{
10465 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10466 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10467 return VINF_SUCCESS;
10468}
10469
10470
10471/**
10472 * VM-exit handler for expiry of the VMX preemption timer.
10473 */
10474HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10475{
10476 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10477
10478 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10479 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10480
10481 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10482 PVM pVM = pVCpu->CTX_SUFF(pVM);
10483 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10484 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10485 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10486}
10487
10488
10489/**
10490 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10491 */
10492HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10493{
10494 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10495
10496 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10497 /** @todo check if XSETBV is supported by the recompiler. */
10498 return VERR_EM_INTERPRETER;
10499}
10500
10501
10502/**
10503 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10504 */
10505HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10506{
10507 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10508
10509 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10510 /** @todo implement EMInterpretInvpcid() */
10511 return VERR_EM_INTERPRETER;
10512}
10513
10514
10515/**
10516 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10517 * Error VM-exit.
10518 */
10519HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10520{
10521 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10522 AssertRCReturn(rc, rc);
10523
10524 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10525 AssertRCReturn(rc, rc);
10526
10527 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10528 NOREF(uInvalidReason);
10529
10530#ifdef VBOX_STRICT
10531 uint32_t uIntrState;
10532 HMVMXHCUINTREG uHCReg;
10533 uint64_t u64Val;
10534 uint32_t u32Val;
10535
10536 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10537 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10538 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10539 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10540 AssertRCReturn(rc, rc);
10541
10542 Log4(("uInvalidReason %u\n", uInvalidReason));
10543 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10544 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10545 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10546 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10547
10548 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10549 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10550 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10551 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10552 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10553 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10554 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10555 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10556 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10557 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10558 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10559 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10560#else
10561 NOREF(pVmxTransient);
10562#endif
10563
10564 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10565 return VERR_VMX_INVALID_GUEST_STATE;
10566}
10567
10568
10569/**
10570 * VM-exit handler for VM-entry failure due to an MSR-load
10571 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10572 */
10573HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10574{
10575 NOREF(pVmxTransient);
10576 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10577 HMVMX_RETURN_UNEXPECTED_EXIT();
10578}
10579
10580
10581/**
10582 * VM-exit handler for VM-entry failure due to a machine-check event
10583 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10584 */
10585HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10586{
10587 NOREF(pVmxTransient);
10588 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10589 HMVMX_RETURN_UNEXPECTED_EXIT();
10590}
10591
10592
10593/**
10594 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10595 * theory.
10596 */
10597HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10598{
10599 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10600 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10601 return VERR_VMX_UNDEFINED_EXIT_CODE;
10602}
10603
10604
10605/**
10606 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10607 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10608 * Conditional VM-exit.
10609 */
10610HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10611{
10612 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10613
10614 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10615 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10616 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10617 return VERR_EM_INTERPRETER;
10618 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10619 HMVMX_RETURN_UNEXPECTED_EXIT();
10620}
10621
10622
10623/**
10624 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10625 */
10626HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10627{
10628 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10629
10630 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10631 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10632 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10633 return VERR_EM_INTERPRETER;
10634 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10635 HMVMX_RETURN_UNEXPECTED_EXIT();
10636}
10637
10638
10639/**
10640 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10641 */
10642HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10643{
10644 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10645
10646 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10647 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10648 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10649 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10650 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10651 {
10652 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10653 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10654 }
10655 AssertRCReturn(rc, rc);
10656 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10657
10658#ifdef VBOX_STRICT
10659 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10660 {
10661 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10662 && pMixedCtx->ecx != MSR_K6_EFER)
10663 {
10664 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10665 HMVMX_RETURN_UNEXPECTED_EXIT();
10666 }
10667# if HC_ARCH_BITS == 64
10668 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10669 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10670 {
10671 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10672 HMVMX_RETURN_UNEXPECTED_EXIT();
10673 }
10674# endif
10675 }
10676#endif
10677
10678 PVM pVM = pVCpu->CTX_SUFF(pVM);
10679 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10680 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10681 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10682 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10683 if (RT_LIKELY(rc == VINF_SUCCESS))
10684 {
10685 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10686 Assert(pVmxTransient->cbInstr == 2);
10687 }
10688 return rc;
10689}
10690
10691
10692/**
10693 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10694 */
10695HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10696{
10697 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10698 PVM pVM = pVCpu->CTX_SUFF(pVM);
10699 int rc = VINF_SUCCESS;
10700
10701 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10702 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10703 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10704 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10705 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10706 {
10707 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10708 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10709 }
10710 AssertRCReturn(rc, rc);
10711 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10712
10713 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10714 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10715 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10716
10717 if (RT_LIKELY(rc == VINF_SUCCESS))
10718 {
10719 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10720
10721 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10722 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10723 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10724 {
10725 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10726 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10727 EMInterpretWrmsr() changes it. */
10728 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10729 }
10730 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10731 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10732 else if (pMixedCtx->ecx == MSR_K6_EFER)
10733 {
10734 /*
10735 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10736 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10737 * the other bits as well, SCE and NXE. See @bugref{7368}.
10738 */
10739 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10740 }
10741
10742 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10743 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10744 {
10745 switch (pMixedCtx->ecx)
10746 {
10747 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10748 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10749 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10750 case MSR_K8_FS_BASE: /* no break */
10751 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10752 case MSR_K6_EFER: /* already handled above */ break;
10753 default:
10754 {
10755 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10756 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10757#if HC_ARCH_BITS == 64
10758 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10759 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10760#endif
10761 break;
10762 }
10763 }
10764 }
10765#ifdef VBOX_STRICT
10766 else
10767 {
10768 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10769 switch (pMixedCtx->ecx)
10770 {
10771 case MSR_IA32_SYSENTER_CS:
10772 case MSR_IA32_SYSENTER_EIP:
10773 case MSR_IA32_SYSENTER_ESP:
10774 case MSR_K8_FS_BASE:
10775 case MSR_K8_GS_BASE:
10776 {
10777 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10778 HMVMX_RETURN_UNEXPECTED_EXIT();
10779 }
10780
10781 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10782 default:
10783 {
10784 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10785 {
10786 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10787 if (pMixedCtx->ecx != MSR_K6_EFER)
10788 {
10789 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10790 pMixedCtx->ecx));
10791 HMVMX_RETURN_UNEXPECTED_EXIT();
10792 }
10793 }
10794
10795#if HC_ARCH_BITS == 64
10796 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10797 {
10798 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10799 HMVMX_RETURN_UNEXPECTED_EXIT();
10800 }
10801#endif
10802 break;
10803 }
10804 }
10805 }
10806#endif /* VBOX_STRICT */
10807 }
10808 return rc;
10809}
10810
10811
10812/**
10813 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10814 */
10815HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10816{
10817 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10818
10819 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10820 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10821 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10822 return VERR_EM_INTERPRETER;
10823 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10824 HMVMX_RETURN_UNEXPECTED_EXIT();
10825}
10826
10827
10828/**
10829 * VM-exit handler for when the TPR value is lowered below the specified
10830 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10831 */
10832HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10833{
10834 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10835 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10836
10837 /*
10838 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10839 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10840 * resume guest execution.
10841 */
10842 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10843 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10844 return VINF_SUCCESS;
10845}
10846
10847
10848/**
10849 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10850 * VM-exit.
10851 *
10852 * @retval VINF_SUCCESS when guest execution can continue.
10853 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10854 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10855 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10856 * recompiler.
10857 */
10858HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10859{
10860 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10861 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10862 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10863 AssertRCReturn(rc, rc);
10864
10865 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10866 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10867 PVM pVM = pVCpu->CTX_SUFF(pVM);
10868 switch (uAccessType)
10869 {
10870 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10871 {
10872#if 0
10873 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10874 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10875#else
10876 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10877 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10878 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10879#endif
10880 AssertRCReturn(rc, rc);
10881
10882 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10883 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10884 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10885 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10886
10887 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10888 {
10889 case 0: /* CR0 */
10890 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10891 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10892 break;
10893 case 2: /* CR2 */
10894 /* Nothing to do here, CR2 it's not part of the VMCS. */
10895 break;
10896 case 3: /* CR3 */
10897 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10898 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10899 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10900 break;
10901 case 4: /* CR4 */
10902 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10903 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10904 break;
10905 case 8: /* CR8 */
10906 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10907 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10908 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10909 break;
10910 default:
10911 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10912 break;
10913 }
10914
10915 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10916 break;
10917 }
10918
10919 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10920 {
10921 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10922 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10923 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10924 AssertRCReturn(rc, rc);
10925 Assert( !pVM->hm.s.fNestedPaging
10926 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10927 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10928
10929 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10930 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10931 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10932
10933 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10934 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10935 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10936 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10937 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10938 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10939 break;
10940 }
10941
10942 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10943 {
10944 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10945 AssertRCReturn(rc, rc);
10946 rc = EMInterpretCLTS(pVM, pVCpu);
10947 AssertRCReturn(rc, rc);
10948 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10949 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10950 Log4(("CRX CLTS write rc=%d\n", rc));
10951 break;
10952 }
10953
10954 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10955 {
10956 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10957 AssertRCReturn(rc, rc);
10958 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10959 if (RT_LIKELY(rc == VINF_SUCCESS))
10960 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10961 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10962 Log4(("CRX LMSW write rc=%d\n", rc));
10963 break;
10964 }
10965
10966 default:
10967 {
10968 AssertMsgFailed(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType));
10969 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10970 }
10971 }
10972
10973 /* Validate possible error codes. */
10974 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10975 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10976 if (RT_SUCCESS(rc))
10977 {
10978 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10979 AssertRCReturn(rc2, rc2);
10980 }
10981 else if (rc == VINF_PGM_CHANGE_MODE)
10982 {
10983 /*
10984 * Clear the exception-mask here rather than messing with it in hmR0VmxLoadSharedCR0(). Since the fRealOnV86Active
10985 * state may be changed now. Re-evaluate the necessary intercepts when we return to VT-x execution via
10986 * hmR0VmxLoadSharedCR0() and hmR0VmxLoadSharedDebugState(), see @bugref{7626}.
10987 */
10988 hmR0VmxInitXcptBitmap(pVM, pVCpu);
10989 }
10990
10991 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10992 return rc;
10993}
10994
10995
10996/**
10997 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10998 * VM-exit.
10999 */
11000HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11001{
11002 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11003 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
11004
11005 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11006 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11007 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11008 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
11009 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
11010 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
11011 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
11012 AssertRCReturn(rc2, rc2);
11013
11014 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
11015 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
11016 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
11017 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
11018 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
11019 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
11020 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11021 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
11022
11023 /* I/O operation lookup arrays. */
11024 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
11025 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
11026
11027 VBOXSTRICTRC rcStrict;
11028 uint32_t const cbValue = s_aIOSizes[uIOWidth];
11029 uint32_t const cbInstr = pVmxTransient->cbInstr;
11030 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
11031 PVM pVM = pVCpu->CTX_SUFF(pVM);
11032 if (fIOString)
11033 {
11034#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
11035 /*
11036 * INS/OUTS - I/O String instruction.
11037 *
11038 * Use instruction-information if available, otherwise fall back on
11039 * interpreting the instruction.
11040 */
11041 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11042 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
11043 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
11044 {
11045 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11046 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11047 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11048 AssertRCReturn(rc2, rc2);
11049 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
11050 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
11051 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
11052 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
11053 if (fIOWrite)
11054 {
11055 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
11056 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
11057 }
11058 else
11059 {
11060 /*
11061 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
11062 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
11063 * See Intel Instruction spec. for "INS".
11064 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
11065 */
11066 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
11067 }
11068 }
11069 else
11070 {
11071 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11072 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11073 AssertRCReturn(rc2, rc2);
11074 rcStrict = IEMExecOne(pVCpu);
11075 }
11076 /** @todo IEM needs to be setting these flags somehow. */
11077 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11078 fUpdateRipAlready = true;
11079#else
11080 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11081 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
11082 if (RT_SUCCESS(rcStrict))
11083 {
11084 if (fIOWrite)
11085 {
11086 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11087 (DISCPUMODE)pDis->uAddrMode, cbValue);
11088 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
11089 }
11090 else
11091 {
11092 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11093 (DISCPUMODE)pDis->uAddrMode, cbValue);
11094 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11095 }
11096 }
11097 else
11098 {
11099 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
11100 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11101 }
11102#endif
11103 }
11104 else
11105 {
11106 /*
11107 * IN/OUT - I/O instruction.
11108 */
11109 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11110 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11111 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11112 if (fIOWrite)
11113 {
11114 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11115 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11116 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11117 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11118 }
11119 else
11120 {
11121 uint32_t u32Result = 0;
11122 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11123 if (IOM_SUCCESS(rcStrict))
11124 {
11125 /* Save result of I/O IN instr. in AL/AX/EAX. */
11126 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11127 }
11128 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11129 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11130 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11131 }
11132 }
11133
11134 if (IOM_SUCCESS(rcStrict))
11135 {
11136 if (!fUpdateRipAlready)
11137 {
11138 pMixedCtx->rip += cbInstr;
11139 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11140 }
11141
11142 /*
11143 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11144 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11145 */
11146 if (fIOString)
11147 {
11148 /** @todo Single-step for INS/OUTS with REP prefix? */
11149 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11150 }
11151 else if (fStepping)
11152 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11153
11154 /*
11155 * If any I/O breakpoints are armed, we need to check if one triggered
11156 * and take appropriate action.
11157 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11158 */
11159 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11160 AssertRCReturn(rc2, rc2);
11161
11162 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11163 * execution engines about whether hyper BPs and such are pending. */
11164 uint32_t const uDr7 = pMixedCtx->dr[7];
11165 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11166 && X86_DR7_ANY_RW_IO(uDr7)
11167 && (pMixedCtx->cr4 & X86_CR4_DE))
11168 || DBGFBpIsHwIoArmed(pVM)))
11169 {
11170 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11171
11172 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11173 VMMRZCallRing3Disable(pVCpu);
11174 HM_DISABLE_PREEMPT_IF_NEEDED();
11175
11176 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11177
11178 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11179 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11180 {
11181 /* Raise #DB. */
11182 if (fIsGuestDbgActive)
11183 ASMSetDR6(pMixedCtx->dr[6]);
11184 if (pMixedCtx->dr[7] != uDr7)
11185 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11186
11187 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11188 }
11189 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11190 else if ( rcStrict2 != VINF_SUCCESS
11191 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11192 rcStrict = rcStrict2;
11193
11194 HM_RESTORE_PREEMPT_IF_NEEDED();
11195 VMMRZCallRing3Enable(pVCpu);
11196 }
11197 }
11198
11199#ifdef DEBUG
11200 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11201 Assert(!fIOWrite);
11202 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11203 Assert(fIOWrite);
11204 else
11205 {
11206 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11207 * statuses, that the VMM device and some others may return. See
11208 * IOM_SUCCESS() for guidance. */
11209 AssertMsg( RT_FAILURE(rcStrict)
11210 || rcStrict == VINF_SUCCESS
11211 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11212 || rcStrict == VINF_EM_DBG_BREAKPOINT
11213 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11214 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11215 }
11216#endif
11217
11218 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11219 return VBOXSTRICTRC_TODO(rcStrict);
11220}
11221
11222
11223/**
11224 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11225 * VM-exit.
11226 */
11227HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11228{
11229 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11230
11231 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11232 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11233 AssertRCReturn(rc, rc);
11234 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11235 {
11236 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11237 AssertRCReturn(rc, rc);
11238 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11239 {
11240 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11241
11242 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11243 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11244
11245 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11246 Assert(!pVCpu->hm.s.Event.fPending);
11247 pVCpu->hm.s.Event.fPending = true;
11248 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11249 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11250 AssertRCReturn(rc, rc);
11251 if (fErrorCodeValid)
11252 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11253 else
11254 pVCpu->hm.s.Event.u32ErrCode = 0;
11255 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11256 && uVector == X86_XCPT_PF)
11257 {
11258 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11259 }
11260
11261 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11262 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11263 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11264 }
11265 }
11266
11267 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11268 * emulation. */
11269 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11270 return VERR_EM_INTERPRETER;
11271}
11272
11273
11274/**
11275 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11276 */
11277HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11278{
11279 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11280 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11281 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11282 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11283 AssertRCReturn(rc, rc);
11284 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11285 return VINF_EM_DBG_STEPPED;
11286}
11287
11288
11289/**
11290 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11291 */
11292HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11293{
11294 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11295
11296 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11297 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11298 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11299 {
11300 if (rc == VINF_HM_DOUBLE_FAULT)
11301 rc = VINF_SUCCESS;
11302 return rc;
11303 }
11304
11305#if 0
11306 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11307 * just sync the whole thing. */
11308 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11309#else
11310 /* Aggressive state sync. for now. */
11311 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11312 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11313 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11314#endif
11315 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11316 AssertRCReturn(rc, rc);
11317
11318 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11319 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11320 switch (uAccessType)
11321 {
11322 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11323 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11324 {
11325 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11326 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11327 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11328
11329 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11330 GCPhys &= PAGE_BASE_GC_MASK;
11331 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11332 PVM pVM = pVCpu->CTX_SUFF(pVM);
11333 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11334 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11335
11336 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11337 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11338 CPUMCTX2CORE(pMixedCtx), GCPhys);
11339 rc = VBOXSTRICTRC_VAL(rc2);
11340 Log4(("ApicAccess rc=%d\n", rc));
11341 if ( rc == VINF_SUCCESS
11342 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11343 || rc == VERR_PAGE_NOT_PRESENT)
11344 {
11345 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11346 | HM_CHANGED_GUEST_RSP
11347 | HM_CHANGED_GUEST_RFLAGS
11348 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11349 rc = VINF_SUCCESS;
11350 }
11351 break;
11352 }
11353
11354 default:
11355 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11356 rc = VINF_EM_RAW_EMULATE_INSTR;
11357 break;
11358 }
11359
11360 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11361 if (rc != VINF_SUCCESS)
11362 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccessToR3);
11363 return rc;
11364}
11365
11366
11367/**
11368 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11369 * VM-exit.
11370 */
11371HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11372{
11373 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11374
11375 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11376 if (pVmxTransient->fWasGuestDebugStateActive)
11377 {
11378 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11379 HMVMX_RETURN_UNEXPECTED_EXIT();
11380 }
11381
11382 int rc = VERR_INTERNAL_ERROR_5;
11383 if ( !DBGFIsStepping(pVCpu)
11384 && !pVCpu->hm.s.fSingleInstruction
11385 && !pVmxTransient->fWasHyperDebugStateActive)
11386 {
11387 /* Don't intercept MOV DRx and #DB any more. */
11388 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11389 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11390 AssertRCReturn(rc, rc);
11391
11392 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11393 {
11394#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11395 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11396 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11397 AssertRCReturn(rc, rc);
11398#endif
11399 }
11400
11401 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11402 VMMRZCallRing3Disable(pVCpu);
11403 HM_DISABLE_PREEMPT_IF_NEEDED();
11404
11405 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11406 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11407 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11408
11409 HM_RESTORE_PREEMPT_IF_NEEDED();
11410 VMMRZCallRing3Enable(pVCpu);
11411
11412#ifdef VBOX_WITH_STATISTICS
11413 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11414 AssertRCReturn(rc, rc);
11415 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11416 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11417 else
11418 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11419#endif
11420 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11421 return VINF_SUCCESS;
11422 }
11423
11424 /*
11425 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11426 * Update the segment registers and DR7 from the CPU.
11427 */
11428 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11429 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11430 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11431 AssertRCReturn(rc, rc);
11432 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11433
11434 PVM pVM = pVCpu->CTX_SUFF(pVM);
11435 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11436 {
11437 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11438 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11439 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11440 if (RT_SUCCESS(rc))
11441 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11442 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11443 }
11444 else
11445 {
11446 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11447 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11448 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11449 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11450 }
11451
11452 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11453 if (RT_SUCCESS(rc))
11454 {
11455 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11456 AssertRCReturn(rc2, rc2);
11457 }
11458 return rc;
11459}
11460
11461
11462/**
11463 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11464 * Conditional VM-exit.
11465 */
11466HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11467{
11468 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11469 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11470
11471 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11472 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11473 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11474 {
11475 if (rc == VINF_HM_DOUBLE_FAULT)
11476 rc = VINF_SUCCESS;
11477 return rc;
11478 }
11479
11480 RTGCPHYS GCPhys = 0;
11481 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11482
11483#if 0
11484 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11485#else
11486 /* Aggressive state sync. for now. */
11487 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11488 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11489 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11490#endif
11491 AssertRCReturn(rc, rc);
11492
11493 /*
11494 * If we succeed, resume guest execution.
11495 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11496 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11497 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11498 * weird case. See @bugref{6043}.
11499 */
11500 PVM pVM = pVCpu->CTX_SUFF(pVM);
11501 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11502 rc = VBOXSTRICTRC_VAL(rc2);
11503 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11504 if ( rc == VINF_SUCCESS
11505 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11506 || rc == VERR_PAGE_NOT_PRESENT)
11507 {
11508 /* Successfully handled MMIO operation. */
11509 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11510 | HM_CHANGED_GUEST_RSP
11511 | HM_CHANGED_GUEST_RFLAGS
11512 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11513 rc = VINF_SUCCESS;
11514 }
11515 return rc;
11516}
11517
11518
11519/**
11520 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11521 * VM-exit.
11522 */
11523HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11524{
11525 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11526 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11527
11528 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11529 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11530 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11531 {
11532 if (rc == VINF_HM_DOUBLE_FAULT)
11533 rc = VINF_SUCCESS;
11534 return rc;
11535 }
11536
11537 RTGCPHYS GCPhys = 0;
11538 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11539 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11540#if 0
11541 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11542#else
11543 /* Aggressive state sync. for now. */
11544 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11545 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11546 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11547#endif
11548 AssertRCReturn(rc, rc);
11549
11550 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11551 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11552
11553 RTGCUINT uErrorCode = 0;
11554 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11555 uErrorCode |= X86_TRAP_PF_ID;
11556 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11557 uErrorCode |= X86_TRAP_PF_RW;
11558 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11559 uErrorCode |= X86_TRAP_PF_P;
11560
11561 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11562
11563 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11564 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11565
11566 /* Handle the pagefault trap for the nested shadow table. */
11567 PVM pVM = pVCpu->CTX_SUFF(pVM);
11568 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11569 TRPMResetTrap(pVCpu);
11570
11571 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11572 if ( rc == VINF_SUCCESS
11573 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11574 || rc == VERR_PAGE_NOT_PRESENT)
11575 {
11576 /* Successfully synced our nested page tables. */
11577 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11578 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11579 | HM_CHANGED_GUEST_RSP
11580 | HM_CHANGED_GUEST_RFLAGS);
11581 return VINF_SUCCESS;
11582 }
11583
11584 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11585 return rc;
11586}
11587
11588/** @} */
11589
11590/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11591/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11592/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11593
11594/** @name VM-exit exception handlers.
11595 * @{
11596 */
11597
11598/**
11599 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11600 */
11601static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11602{
11603 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11604 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11605
11606 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11607 AssertRCReturn(rc, rc);
11608
11609 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11610 {
11611 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11612 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11613
11614 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11615 * provides VM-exit instruction length. If this causes problem later,
11616 * disassemble the instruction like it's done on AMD-V. */
11617 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11618 AssertRCReturn(rc2, rc2);
11619 return rc;
11620 }
11621
11622 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11623 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11624 return rc;
11625}
11626
11627
11628/**
11629 * VM-exit exception handler for #BP (Breakpoint exception).
11630 */
11631static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11632{
11633 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11634 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11635
11636 /** @todo Try optimize this by not saving the entire guest state unless
11637 * really needed. */
11638 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11639 AssertRCReturn(rc, rc);
11640
11641 PVM pVM = pVCpu->CTX_SUFF(pVM);
11642 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11643 if (rc == VINF_EM_RAW_GUEST_TRAP)
11644 {
11645 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11646 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11647 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11648 AssertRCReturn(rc, rc);
11649
11650 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11651 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11652 }
11653
11654 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11655 return rc;
11656}
11657
11658
11659/**
11660 * VM-exit exception handler for #DB (Debug exception).
11661 */
11662static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11663{
11664 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11665 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11666 Log6(("XcptDB\n"));
11667
11668 /*
11669 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11670 * for processing.
11671 */
11672 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11673 AssertRCReturn(rc, rc);
11674
11675 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11676 uint64_t uDR6 = X86_DR6_INIT_VAL;
11677 uDR6 |= ( pVmxTransient->uExitQualification
11678 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11679
11680 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11681 if (rc == VINF_EM_RAW_GUEST_TRAP)
11682 {
11683 /*
11684 * The exception was for the guest. Update DR6, DR7.GD and
11685 * IA32_DEBUGCTL.LBR before forwarding it.
11686 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11687 */
11688 VMMRZCallRing3Disable(pVCpu);
11689 HM_DISABLE_PREEMPT_IF_NEEDED();
11690
11691 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11692 pMixedCtx->dr[6] |= uDR6;
11693 if (CPUMIsGuestDebugStateActive(pVCpu))
11694 ASMSetDR6(pMixedCtx->dr[6]);
11695
11696 HM_RESTORE_PREEMPT_IF_NEEDED();
11697 VMMRZCallRing3Enable(pVCpu);
11698
11699 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11700 AssertRCReturn(rc, rc);
11701
11702 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11703 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11704
11705 /* Paranoia. */
11706 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11707 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11708
11709 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11710 AssertRCReturn(rc, rc);
11711
11712 /*
11713 * Raise #DB in the guest.
11714 *
11715 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11716 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11717 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11718 *
11719 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11720 */
11721 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11722 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11723 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11724 AssertRCReturn(rc, rc);
11725 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11726 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11727 return VINF_SUCCESS;
11728 }
11729
11730 /*
11731 * Not a guest trap, must be a hypervisor related debug event then.
11732 * Update DR6 in case someone is interested in it.
11733 */
11734 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11735 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11736 CPUMSetHyperDR6(pVCpu, uDR6);
11737
11738 return rc;
11739}
11740
11741
11742/**
11743 * VM-exit exception handler for #NM (Device-not-available exception: floating
11744 * point exception).
11745 */
11746static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11747{
11748 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11749
11750 /* We require CR0 and EFER. EFER is always up-to-date. */
11751 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11752 AssertRCReturn(rc, rc);
11753
11754 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11755 VMMRZCallRing3Disable(pVCpu);
11756 HM_DISABLE_PREEMPT_IF_NEEDED();
11757
11758 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11759 if (pVmxTransient->fWasGuestFPUStateActive)
11760 {
11761 rc = VINF_EM_RAW_GUEST_TRAP;
11762 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11763 }
11764 else
11765 {
11766#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11767 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11768#endif
11769 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11770 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11771 }
11772
11773 HM_RESTORE_PREEMPT_IF_NEEDED();
11774 VMMRZCallRing3Enable(pVCpu);
11775
11776 if (rc == VINF_SUCCESS)
11777 {
11778 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11779 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11780 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11781 pVCpu->hm.s.fUseGuestFpu = true;
11782 }
11783 else
11784 {
11785 /* Forward #NM to the guest. */
11786 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11787 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11788 AssertRCReturn(rc, rc);
11789 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11790 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11791 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11792 }
11793
11794 return VINF_SUCCESS;
11795}
11796
11797
11798/**
11799 * VM-exit exception handler for #GP (General-protection exception).
11800 *
11801 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11802 */
11803static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11804{
11805 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11806 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11807
11808 int rc = VERR_INTERNAL_ERROR_5;
11809 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11810 {
11811#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11812 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11813 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11814 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11815 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11816 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11817 AssertRCReturn(rc, rc);
11818 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11819 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11820 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11821 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11822 return rc;
11823#else
11824 /* We don't intercept #GP. */
11825 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11826 NOREF(pVmxTransient);
11827 return VERR_VMX_UNEXPECTED_EXCEPTION;
11828#endif
11829 }
11830
11831 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11832 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11833
11834 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11835 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11836 AssertRCReturn(rc, rc);
11837
11838 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11839 uint32_t cbOp = 0;
11840 PVM pVM = pVCpu->CTX_SUFF(pVM);
11841 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11842 if (RT_SUCCESS(rc))
11843 {
11844 rc = VINF_SUCCESS;
11845 Assert(cbOp == pDis->cbInstr);
11846 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11847 switch (pDis->pCurInstr->uOpcode)
11848 {
11849 case OP_CLI:
11850 {
11851 pMixedCtx->eflags.Bits.u1IF = 0;
11852 pMixedCtx->eflags.Bits.u1RF = 0;
11853 pMixedCtx->rip += pDis->cbInstr;
11854 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11855 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11857 break;
11858 }
11859
11860 case OP_STI:
11861 {
11862 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11863 pMixedCtx->eflags.Bits.u1IF = 1;
11864 pMixedCtx->eflags.Bits.u1RF = 0;
11865 pMixedCtx->rip += pDis->cbInstr;
11866 if (!fOldIF)
11867 {
11868 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11869 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11870 }
11871 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11872 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11873 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11874 break;
11875 }
11876
11877 case OP_HLT:
11878 {
11879 rc = VINF_EM_HALT;
11880 pMixedCtx->rip += pDis->cbInstr;
11881 pMixedCtx->eflags.Bits.u1RF = 0;
11882 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11883 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11884 break;
11885 }
11886
11887 case OP_POPF:
11888 {
11889 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11890 uint32_t cbParm;
11891 uint32_t uMask;
11892 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11893 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11894 {
11895 cbParm = 4;
11896 uMask = 0xffffffff;
11897 }
11898 else
11899 {
11900 cbParm = 2;
11901 uMask = 0xffff;
11902 }
11903
11904 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11905 RTGCPTR GCPtrStack = 0;
11906 X86EFLAGS Eflags;
11907 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11908 &GCPtrStack);
11909 if (RT_SUCCESS(rc))
11910 {
11911 Assert(sizeof(Eflags.u32) >= cbParm);
11912 Eflags.u32 = 0;
11913 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11914 }
11915 if (RT_FAILURE(rc))
11916 {
11917 rc = VERR_EM_INTERPRETER;
11918 break;
11919 }
11920 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11921 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
11922 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11923 pMixedCtx->esp += cbParm;
11924 pMixedCtx->esp &= uMask;
11925 pMixedCtx->rip += pDis->cbInstr;
11926 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11927 | HM_CHANGED_GUEST_RSP
11928 | HM_CHANGED_GUEST_RFLAGS);
11929 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11930 if (fStepping)
11931 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11932
11933 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11934 break;
11935 }
11936
11937 case OP_PUSHF:
11938 {
11939 uint32_t cbParm;
11940 uint32_t uMask;
11941 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11942 {
11943 cbParm = 4;
11944 uMask = 0xffffffff;
11945 }
11946 else
11947 {
11948 cbParm = 2;
11949 uMask = 0xffff;
11950 }
11951
11952 /* Get the stack pointer & push the contents of eflags onto the stack. */
11953 RTGCPTR GCPtrStack = 0;
11954 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11955 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11956 if (RT_FAILURE(rc))
11957 {
11958 rc = VERR_EM_INTERPRETER;
11959 break;
11960 }
11961 X86EFLAGS Eflags = pMixedCtx->eflags;
11962 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11963 Eflags.Bits.u1RF = 0;
11964 Eflags.Bits.u1VM = 0;
11965
11966 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11967 if (RT_FAILURE(rc))
11968 {
11969 rc = VERR_EM_INTERPRETER;
11970 break;
11971 }
11972 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11973 pMixedCtx->esp -= cbParm;
11974 pMixedCtx->esp &= uMask;
11975 pMixedCtx->rip += pDis->cbInstr;
11976 pMixedCtx->eflags.Bits.u1RF = 0;
11977 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11978 | HM_CHANGED_GUEST_RSP
11979 | HM_CHANGED_GUEST_RFLAGS);
11980 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11981 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11982 break;
11983 }
11984
11985 case OP_IRET:
11986 {
11987 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11988 * instruction reference. */
11989 RTGCPTR GCPtrStack = 0;
11990 uint32_t uMask = 0xffff;
11991 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11992 uint16_t aIretFrame[3];
11993 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11994 {
11995 rc = VERR_EM_INTERPRETER;
11996 break;
11997 }
11998 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11999 &GCPtrStack);
12000 if (RT_SUCCESS(rc))
12001 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
12002 if (RT_FAILURE(rc))
12003 {
12004 rc = VERR_EM_INTERPRETER;
12005 break;
12006 }
12007 pMixedCtx->eip = 0;
12008 pMixedCtx->ip = aIretFrame[0];
12009 pMixedCtx->cs.Sel = aIretFrame[1];
12010 pMixedCtx->cs.ValidSel = aIretFrame[1];
12011 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
12012 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
12013 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
12014 pMixedCtx->sp += sizeof(aIretFrame);
12015 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12016 | HM_CHANGED_GUEST_SEGMENT_REGS
12017 | HM_CHANGED_GUEST_RSP
12018 | HM_CHANGED_GUEST_RFLAGS);
12019 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
12020 if (fStepping)
12021 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
12022 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
12023 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
12024 break;
12025 }
12026
12027 case OP_INT:
12028 {
12029 uint16_t uVector = pDis->Param1.uValue & 0xff;
12030 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
12031 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12032 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12033 break;
12034 }
12035
12036 case OP_INTO:
12037 {
12038 if (pMixedCtx->eflags.Bits.u1OF)
12039 {
12040 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
12041 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12042 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12043 }
12044 else
12045 {
12046 pMixedCtx->eflags.Bits.u1RF = 0;
12047 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12048 }
12049 break;
12050 }
12051
12052 default:
12053 {
12054 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
12055 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
12056 EMCODETYPE_SUPERVISOR);
12057 rc = VBOXSTRICTRC_VAL(rc2);
12058 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
12059 /** @todo We have to set pending-debug exceptions here when the guest is
12060 * single-stepping depending on the instruction that was interpreted. */
12061 Log4(("#GP rc=%Rrc\n", rc));
12062 break;
12063 }
12064 }
12065 }
12066 else
12067 rc = VERR_EM_INTERPRETER;
12068
12069 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
12070 ("#GP Unexpected rc=%Rrc\n", rc));
12071 return rc;
12072}
12073
12074
12075#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12076/**
12077 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
12078 * the exception reported in the VMX transient structure back into the VM.
12079 *
12080 * @remarks Requires uExitIntInfo in the VMX transient structure to be
12081 * up-to-date.
12082 */
12083static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12084{
12085 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12086
12087 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12088 hmR0VmxCheckExitDueToEventDelivery(). */
12089 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12090 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12091 AssertRCReturn(rc, rc);
12092 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12093
12094#ifdef DEBUG_ramshankar
12095 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12096 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12097 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12098#endif
12099
12100 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12101 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12102 return VINF_SUCCESS;
12103}
12104#endif
12105
12106
12107/**
12108 * VM-exit exception handler for #PF (Page-fault exception).
12109 */
12110static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12111{
12112 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12113 PVM pVM = pVCpu->CTX_SUFF(pVM);
12114 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12115 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12116 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12117 AssertRCReturn(rc, rc);
12118
12119#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12120 if (pVM->hm.s.fNestedPaging)
12121 {
12122 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12123 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12124 {
12125 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12126 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12127 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12128 }
12129 else
12130 {
12131 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12132 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12133 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12134 }
12135 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12136 return rc;
12137 }
12138#else
12139 Assert(!pVM->hm.s.fNestedPaging);
12140 NOREF(pVM);
12141#endif
12142
12143 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12144 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12145 if (pVmxTransient->fVectoringPF)
12146 {
12147 Assert(pVCpu->hm.s.Event.fPending);
12148 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12149 }
12150
12151 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12152 AssertRCReturn(rc, rc);
12153
12154 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12155 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12156
12157 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12158 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12159 (RTGCPTR)pVmxTransient->uExitQualification);
12160
12161 Log4(("#PF: rc=%Rrc\n", rc));
12162 if (rc == VINF_SUCCESS)
12163 {
12164 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12165 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12166 * memory? We don't update the whole state here... */
12167 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12168 | HM_CHANGED_GUEST_RSP
12169 | HM_CHANGED_GUEST_RFLAGS
12170 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12171 TRPMResetTrap(pVCpu);
12172 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12173 return rc;
12174 }
12175
12176 if (rc == VINF_EM_RAW_GUEST_TRAP)
12177 {
12178 if (!pVmxTransient->fVectoringDoublePF)
12179 {
12180 /* It's a guest page fault and needs to be reflected to the guest. */
12181 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12182 TRPMResetTrap(pVCpu);
12183 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12184 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12185 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12186 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12187 }
12188 else
12189 {
12190 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12191 TRPMResetTrap(pVCpu);
12192 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12193 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12194 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12195 }
12196
12197 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12198 return VINF_SUCCESS;
12199 }
12200
12201 TRPMResetTrap(pVCpu);
12202 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12203 return rc;
12204}
12205
12206/** @} */
12207
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette