VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 58156

Last change on this file since 58156 was 58126, checked in by vboxsync, 9 years ago

VMM: Fixed almost all the Doxygen warnings.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 508.5 KB
Line 
1/* $Id: HMVMXR0.cpp 58126 2015-10-08 20:59:48Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#include "HMInternal.h"
38#include <VBox/vmm/vm.h>
39#include "HMVMXR0.h"
40#include "dtrace/VBoxVMM.h"
41
42#ifdef DEBUG_ramshankar
43# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
44# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_CHECK_GUEST_STATE
47# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
48# define HMVMX_ALWAYS_TRAP_PF
49# define HMVMX_ALWAYS_SWAP_FPU_STATE
50# define HMVMX_ALWAYS_FLUSH_TLB
51# define HMVMX_ALWAYS_SWAP_EFER
52#endif
53
54
55/*********************************************************************************************************************************
56* Defined Constants And Macros *
57*********************************************************************************************************************************/
58/** Use the function table. */
59#define HMVMX_USE_FUNCTION_TABLE
60
61/** Determine which tagged-TLB flush handler to use. */
62#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
63#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
64#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
65#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
66
67/** @name Updated-guest-state flags.
68 * @{ */
69#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
70#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
71#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
72#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
73#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
74#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
75#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
76#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
77#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
78#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
79#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
80#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
81#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
82#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
83#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
84#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
85#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
86#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
87#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
88#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
89#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
90 | HMVMX_UPDATED_GUEST_RSP \
91 | HMVMX_UPDATED_GUEST_RFLAGS \
92 | HMVMX_UPDATED_GUEST_CR0 \
93 | HMVMX_UPDATED_GUEST_CR3 \
94 | HMVMX_UPDATED_GUEST_CR4 \
95 | HMVMX_UPDATED_GUEST_GDTR \
96 | HMVMX_UPDATED_GUEST_IDTR \
97 | HMVMX_UPDATED_GUEST_LDTR \
98 | HMVMX_UPDATED_GUEST_TR \
99 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
100 | HMVMX_UPDATED_GUEST_DEBUG \
101 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
102 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
104 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
105 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
106 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
107 | HMVMX_UPDATED_GUEST_INTR_STATE \
108 | HMVMX_UPDATED_GUEST_APIC_STATE)
109/** @} */
110
111/** @name
112 * Flags to skip redundant reads of some common VMCS fields that are not part of
113 * the guest-CPU state but are in the transient structure.
114 */
115#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
117#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
122/** @} */
123
124/** @name
125 * States of the VMCS.
126 *
127 * This does not reflect all possible VMCS states but currently only those
128 * needed for maintaining the VMCS consistently even when thread-context hooks
129 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
130 */
131#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
132#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
133#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
134/** @} */
135
136/**
137 * Exception bitmap mask for real-mode guests (real-on-v86).
138 *
139 * We need to intercept all exceptions manually except:
140 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
141 * - \#DB handled in hmR0VmxLoadSharedDebugState().
142 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
143 * support.
144 */
145#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
146 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
147 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
148 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
149 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
150 /* RT_BIT(X86_XCPT_MF) */ | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
151 | RT_BIT(X86_XCPT_XF))
152
153/**
154 * Exception bitmap mask for all contributory exceptions.
155 *
156 * Page fault is deliberately excluded here as it's conditional as to whether
157 * it's contributory or benign. Page faults are handled separately.
158 */
159#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
160 | RT_BIT(X86_XCPT_DE))
161
162/** Maximum VM-instruction error number. */
163#define HMVMX_INSTR_ERROR_MAX 28
164
165/** Profiling macro. */
166#ifdef HM_PROFILE_EXIT_DISPATCH
167# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
168# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
169#else
170# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
171# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
172#endif
173
174/** Assert that preemption is disabled or covered by thread-context hooks. */
175#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
176 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
177
178/** Assert that we haven't migrated CPUs when thread-context hooks are not
179 * used. */
180#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
181 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
182 ("Illegal migration! Entered on CPU %u Current %u\n", \
183 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
184
185/** Helper macro for VM-exit handlers called unexpectedly. */
186#define HMVMX_RETURN_UNEXPECTED_EXIT() \
187 do { \
188 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
189 return VERR_VMX_UNEXPECTED_EXIT; \
190 } while (0)
191
192
193/*********************************************************************************************************************************
194* Structures and Typedefs *
195*********************************************************************************************************************************/
196/**
197 * VMX transient state.
198 *
199 * A state structure for holding miscellaneous information across
200 * VMX non-root operation and restored after the transition.
201 */
202typedef struct VMXTRANSIENT
203{
204 /** The host's rflags/eflags. */
205 RTCCUINTREG fEFlags;
206#if HC_ARCH_BITS == 32
207 uint32_t u32Alignment0;
208#endif
209 /** The guest's TPR value used for TPR shadowing. */
210 uint8_t u8GuestTpr;
211 /** Alignment. */
212 uint8_t abAlignment0[7];
213
214 /** The basic VM-exit reason. */
215 uint16_t uExitReason;
216 /** Alignment. */
217 uint16_t u16Alignment0;
218 /** The VM-exit interruption error code. */
219 uint32_t uExitIntErrorCode;
220 /** The VM-exit exit code qualification. */
221 uint64_t uExitQualification;
222
223 /** The VM-exit interruption-information field. */
224 uint32_t uExitIntInfo;
225 /** The VM-exit instruction-length field. */
226 uint32_t cbInstr;
227 /** The VM-exit instruction-information field. */
228 union
229 {
230 /** Plain unsigned int representation. */
231 uint32_t u;
232 /** INS and OUTS information. */
233 struct
234 {
235 uint32_t u6Reserved0 : 7;
236 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
237 uint32_t u3AddrSize : 3;
238 uint32_t u5Reserved1 : 5;
239 /** The segment register (X86_SREG_XXX). */
240 uint32_t iSegReg : 3;
241 uint32_t uReserved2 : 14;
242 } StrIo;
243 } ExitInstrInfo;
244 /** Whether the VM-entry failed or not. */
245 bool fVMEntryFailed;
246 /** Alignment. */
247 uint8_t abAlignment1[3];
248
249 /** The VM-entry interruption-information field. */
250 uint32_t uEntryIntInfo;
251 /** The VM-entry exception error code field. */
252 uint32_t uEntryXcptErrorCode;
253 /** The VM-entry instruction length field. */
254 uint32_t cbEntryInstr;
255
256 /** IDT-vectoring information field. */
257 uint32_t uIdtVectoringInfo;
258 /** IDT-vectoring error code. */
259 uint32_t uIdtVectoringErrorCode;
260
261 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
262 uint32_t fVmcsFieldsRead;
263
264 /** Whether the guest FPU was active at the time of VM-exit. */
265 bool fWasGuestFPUStateActive;
266 /** Whether the guest debug state was active at the time of VM-exit. */
267 bool fWasGuestDebugStateActive;
268 /** Whether the hyper debug state was active at the time of VM-exit. */
269 bool fWasHyperDebugStateActive;
270 /** Whether TSC-offsetting should be setup before VM-entry. */
271 bool fUpdateTscOffsettingAndPreemptTimer;
272 /** Whether the VM-exit was caused by a page-fault during delivery of a
273 * contributory exception or a page-fault. */
274 bool fVectoringDoublePF;
275 /** Whether the VM-exit was caused by a page-fault during delivery of an
276 * external interrupt or NMI. */
277 bool fVectoringPF;
278} VMXTRANSIENT;
279AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284/** Pointer to VMX transient state. */
285typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288/**
289 * MSR-bitmap read permissions.
290 */
291typedef enum VMXMSREXITREAD
292{
293 /** Reading this MSR causes a VM-exit. */
294 VMXMSREXIT_INTERCEPT_READ = 0xb,
295 /** Reading this MSR does not cause a VM-exit. */
296 VMXMSREXIT_PASSTHRU_READ
297} VMXMSREXITREAD;
298/** Pointer to MSR-bitmap read permissions. */
299typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301/**
302 * MSR-bitmap write permissions.
303 */
304typedef enum VMXMSREXITWRITE
305{
306 /** Writing to this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308 /** Writing to this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_WRITE
310} VMXMSREXITWRITE;
311/** Pointer to MSR-bitmap write permissions. */
312typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315/**
316 * VMX VM-exit handler.
317 *
318 * @returns VBox status code.
319 * @param pVCpu The cross context virtual CPU structure.
320 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
321 * out-of-sync. Make sure to update the required
322 * fields before using them.
323 * @param pVmxTransient Pointer to the VMX-transient structure.
324 */
325#ifndef HMVMX_USE_FUNCTION_TABLE
326typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327#else
328typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329/** Pointer to VM-exit handler. */
330typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331#endif
332
333
334/*********************************************************************************************************************************
335* Internal Functions *
336*********************************************************************************************************************************/
337static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
338static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
339static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
340static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
341 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
342 bool fStepping, uint32_t *puIntState);
343#if HC_ARCH_BITS == 32
344static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
345#endif
346#ifndef HMVMX_USE_FUNCTION_TABLE
347DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
348# define HMVMX_EXIT_DECL static int
349#else
350# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
351#endif
352DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
353 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
354
355/** @name VM-exit handlers.
356 * @{
357 */
358static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
359static FNVMXEXITHANDLER hmR0VmxExitExtInt;
360static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
361static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
362static FNVMXEXITHANDLER hmR0VmxExitSipi;
363static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
364static FNVMXEXITHANDLER hmR0VmxExitSmi;
365static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
366static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
367static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
368static FNVMXEXITHANDLER hmR0VmxExitCpuid;
369static FNVMXEXITHANDLER hmR0VmxExitGetsec;
370static FNVMXEXITHANDLER hmR0VmxExitHlt;
371static FNVMXEXITHANDLER hmR0VmxExitInvd;
372static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
373static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
374static FNVMXEXITHANDLER hmR0VmxExitVmcall;
375static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
376static FNVMXEXITHANDLER hmR0VmxExitRsm;
377static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
378static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
379static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
380static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
381static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
382static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
383static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
384static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
385static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
386static FNVMXEXITHANDLER hmR0VmxExitMwait;
387static FNVMXEXITHANDLER hmR0VmxExitMtf;
388static FNVMXEXITHANDLER hmR0VmxExitMonitor;
389static FNVMXEXITHANDLER hmR0VmxExitPause;
390static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
391static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
392static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
393static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
394static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
395static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
396static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
397static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
398static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
399static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
400static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
401static FNVMXEXITHANDLER hmR0VmxExitRdrand;
402static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
403/** @} */
404
405static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
412static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413#endif
414static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
415
416
417/*********************************************************************************************************************************
418* Global Variables *
419*********************************************************************************************************************************/
420#ifdef HMVMX_USE_FUNCTION_TABLE
421
422/**
423 * VMX_EXIT dispatch table.
424 */
425static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
426{
427 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
428 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
429 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
430 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
431 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
432 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
433 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
434 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
435 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
436 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
437 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
438 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
439 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
440 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
441 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
442 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
443 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
444 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
445 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
446 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
447 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
448 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
449 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
450 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
451 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
452 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
453 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
454 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
455 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
456 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
457 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
458 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
459 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
460 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
461 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
462 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
463 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
464 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
465 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
466 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
467 /* 40 UNDEFINED */ hmR0VmxExitPause,
468 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
469 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
470 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
471 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
472 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
473 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
474 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
475 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
476 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
477 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
478 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
479 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
480 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
481 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
482 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
483 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
484 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
485 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
486 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
487 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
488 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
489 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
490 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
491 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
492};
493#endif /* HMVMX_USE_FUNCTION_TABLE */
494
495#ifdef VBOX_STRICT
496static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
497{
498 /* 0 */ "(Not Used)",
499 /* 1 */ "VMCALL executed in VMX root operation.",
500 /* 2 */ "VMCLEAR with invalid physical address.",
501 /* 3 */ "VMCLEAR with VMXON pointer.",
502 /* 4 */ "VMLAUNCH with non-clear VMCS.",
503 /* 5 */ "VMRESUME with non-launched VMCS.",
504 /* 6 */ "VMRESUME after VMXOFF",
505 /* 7 */ "VM-entry with invalid control fields.",
506 /* 8 */ "VM-entry with invalid host state fields.",
507 /* 9 */ "VMPTRLD with invalid physical address.",
508 /* 10 */ "VMPTRLD with VMXON pointer.",
509 /* 11 */ "VMPTRLD with incorrect revision identifier.",
510 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
511 /* 13 */ "VMWRITE to read-only VMCS component.",
512 /* 14 */ "(Not Used)",
513 /* 15 */ "VMXON executed in VMX root operation.",
514 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
515 /* 17 */ "VM-entry with non-launched executing VMCS.",
516 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
517 /* 19 */ "VMCALL with non-clear VMCS.",
518 /* 20 */ "VMCALL with invalid VM-exit control fields.",
519 /* 21 */ "(Not Used)",
520 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
521 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
522 /* 24 */ "VMCALL with invalid SMM-monitor features.",
523 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
524 /* 26 */ "VM-entry with events blocked by MOV SS.",
525 /* 27 */ "(Not Used)",
526 /* 28 */ "Invalid operand to INVEPT/INVVPID."
527};
528#endif /* VBOX_STRICT */
529
530
531
532/**
533 * Updates the VM's last error record.
534 *
535 * If there was a VMX instruction error, reads the error data from the VMCS and
536 * updates VCPU's last error record as well.
537 *
538 * @param pVM The cross context VM structure.
539 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
540 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
541 * VERR_VMX_INVALID_VMCS_FIELD.
542 * @param rc The error code.
543 */
544static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
545{
546 AssertPtr(pVM);
547 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
548 || rc == VERR_VMX_UNABLE_TO_START_VM)
549 {
550 AssertPtrReturnVoid(pVCpu);
551 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
552 }
553 pVM->hm.s.lLastError = rc;
554}
555
556
557/**
558 * Reads the VM-entry interruption-information field from the VMCS into the VMX
559 * transient structure.
560 *
561 * @returns VBox status code.
562 * @param pVmxTransient Pointer to the VMX transient structure.
563 *
564 * @remarks No-long-jump zone!!!
565 */
566DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
567{
568 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
569 AssertRCReturn(rc, rc);
570 return VINF_SUCCESS;
571}
572
573
574/**
575 * Reads the VM-entry exception error code field from the VMCS into
576 * the VMX transient structure.
577 *
578 * @returns VBox status code.
579 * @param pVmxTransient Pointer to the VMX transient structure.
580 *
581 * @remarks No-long-jump zone!!!
582 */
583DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
584{
585 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
586 AssertRCReturn(rc, rc);
587 return VINF_SUCCESS;
588}
589
590
591/**
592 * Reads the VM-entry exception error code field from the VMCS into
593 * the VMX transient structure.
594 *
595 * @returns VBox status code.
596 * @param pVmxTransient Pointer to the VMX transient structure.
597 *
598 * @remarks No-long-jump zone!!!
599 */
600DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
601{
602 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
603 AssertRCReturn(rc, rc);
604 return VINF_SUCCESS;
605}
606
607
608/**
609 * Reads the VM-exit interruption-information field from the VMCS into the VMX
610 * transient structure.
611 *
612 * @returns VBox status code.
613 * @param pVmxTransient Pointer to the VMX transient structure.
614 */
615DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
616{
617 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
618 {
619 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
620 AssertRCReturn(rc, rc);
621 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
622 }
623 return VINF_SUCCESS;
624}
625
626
627/**
628 * Reads the VM-exit interruption error code from the VMCS into the VMX
629 * transient structure.
630 *
631 * @returns VBox status code.
632 * @param pVmxTransient Pointer to the VMX transient structure.
633 */
634DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
635{
636 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
637 {
638 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
639 AssertRCReturn(rc, rc);
640 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
641 }
642 return VINF_SUCCESS;
643}
644
645
646/**
647 * Reads the VM-exit instruction length field from the VMCS into the VMX
648 * transient structure.
649 *
650 * @returns VBox status code.
651 * @param pVmxTransient Pointer to the VMX transient structure.
652 */
653DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
654{
655 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
656 {
657 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
658 AssertRCReturn(rc, rc);
659 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
660 }
661 return VINF_SUCCESS;
662}
663
664
665/**
666 * Reads the VM-exit instruction-information field from the VMCS into
667 * the VMX transient structure.
668 *
669 * @returns VBox status code.
670 * @param pVmxTransient Pointer to the VMX transient structure.
671 */
672DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
673{
674 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
675 {
676 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
677 AssertRCReturn(rc, rc);
678 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
679 }
680 return VINF_SUCCESS;
681}
682
683
684/**
685 * Reads the exit code qualification from the VMCS into the VMX transient
686 * structure.
687 *
688 * @returns VBox status code.
689 * @param pVCpu The cross context virtual CPU structure of the
690 * calling EMT. (Required for the VMCS cache case.)
691 * @param pVmxTransient Pointer to the VMX transient structure.
692 */
693DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
694{
695 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
696 {
697 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
698 AssertRCReturn(rc, rc);
699 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
700 }
701 return VINF_SUCCESS;
702}
703
704
705/**
706 * Reads the IDT-vectoring information field from the VMCS into the VMX
707 * transient structure.
708 *
709 * @returns VBox status code.
710 * @param pVmxTransient Pointer to the VMX transient structure.
711 *
712 * @remarks No-long-jump zone!!!
713 */
714DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
715{
716 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
717 {
718 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
719 AssertRCReturn(rc, rc);
720 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
721 }
722 return VINF_SUCCESS;
723}
724
725
726/**
727 * Reads the IDT-vectoring error code from the VMCS into the VMX
728 * transient structure.
729 *
730 * @returns VBox status code.
731 * @param pVmxTransient Pointer to the VMX transient structure.
732 */
733DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
734{
735 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
736 {
737 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
738 AssertRCReturn(rc, rc);
739 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
740 }
741 return VINF_SUCCESS;
742}
743
744
745/**
746 * Enters VMX root mode operation on the current CPU.
747 *
748 * @returns VBox status code.
749 * @param pVM The cross context VM structure. Can be
750 * NULL, after a resume.
751 * @param HCPhysCpuPage Physical address of the VMXON region.
752 * @param pvCpuPage Pointer to the VMXON region.
753 */
754static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
755{
756 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
757 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
758 Assert(pvCpuPage);
759 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
760
761 if (pVM)
762 {
763 /* Write the VMCS revision dword to the VMXON region. */
764 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
765 }
766
767 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
768 RTCCUINTREG fEFlags = ASMIntDisableFlags();
769
770 /* Enable the VMX bit in CR4 if necessary. */
771 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
772
773 /* Enter VMX root mode. */
774 int rc = VMXEnable(HCPhysCpuPage);
775 if (RT_FAILURE(rc))
776 {
777 if (!(uOldCr4 & X86_CR4_VMXE))
778 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
779
780 if (pVM)
781 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
782 }
783
784 /* Restore interrupts. */
785 ASMSetFlags(fEFlags);
786 return rc;
787}
788
789
790/**
791 * Exits VMX root mode operation on the current CPU.
792 *
793 * @returns VBox status code.
794 */
795static int hmR0VmxLeaveRootMode(void)
796{
797 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
798
799 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
800 RTCCUINTREG fEFlags = ASMIntDisableFlags();
801
802 /* If we're for some reason not in VMX root mode, then don't leave it. */
803 RTCCUINTREG uHostCR4 = ASMGetCR4();
804
805 int rc;
806 if (uHostCR4 & X86_CR4_VMXE)
807 {
808 /* Exit VMX root mode and clear the VMX bit in CR4. */
809 VMXDisable();
810 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
811 rc = VINF_SUCCESS;
812 }
813 else
814 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
815
816 /* Restore interrupts. */
817 ASMSetFlags(fEFlags);
818 return rc;
819}
820
821
822/**
823 * Allocates and maps one physically contiguous page. The allocated page is
824 * zero'd out. (Used by various VT-x structures).
825 *
826 * @returns IPRT status code.
827 * @param pMemObj Pointer to the ring-0 memory object.
828 * @param ppVirt Where to store the virtual address of the
829 * allocation.
830 * @param pHCPhys Where to store the physical address of the
831 * allocation.
832 */
833DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
834{
835 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
836 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
837 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
838
839 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
840 if (RT_FAILURE(rc))
841 return rc;
842 *ppVirt = RTR0MemObjAddress(*pMemObj);
843 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
844 ASMMemZero32(*ppVirt, PAGE_SIZE);
845 return VINF_SUCCESS;
846}
847
848
849/**
850 * Frees and unmaps an allocated physical page.
851 *
852 * @param pMemObj Pointer to the ring-0 memory object.
853 * @param ppVirt Where to re-initialize the virtual address of
854 * allocation as 0.
855 * @param pHCPhys Where to re-initialize the physical address of the
856 * allocation as 0.
857 */
858DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
859{
860 AssertPtr(pMemObj);
861 AssertPtr(ppVirt);
862 AssertPtr(pHCPhys);
863 if (*pMemObj != NIL_RTR0MEMOBJ)
864 {
865 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
866 AssertRC(rc);
867 *pMemObj = NIL_RTR0MEMOBJ;
868 *ppVirt = 0;
869 *pHCPhys = 0;
870 }
871}
872
873
874/**
875 * Worker function to free VT-x related structures.
876 *
877 * @returns IPRT status code.
878 * @param pVM The cross context VM structure.
879 */
880static void hmR0VmxStructsFree(PVM pVM)
881{
882 for (VMCPUID i = 0; i < pVM->cCpus; i++)
883 {
884 PVMCPU pVCpu = &pVM->aCpus[i];
885 AssertPtr(pVCpu);
886
887 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
888 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
889
890 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
891 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
892
893 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
894 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
895 }
896
897 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
898#ifdef VBOX_WITH_CRASHDUMP_MAGIC
899 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
900#endif
901}
902
903
904/**
905 * Worker function to allocate VT-x related VM structures.
906 *
907 * @returns IPRT status code.
908 * @param pVM The cross context VM structure.
909 */
910static int hmR0VmxStructsAlloc(PVM pVM)
911{
912 /*
913 * Initialize members up-front so we can cleanup properly on allocation failure.
914 */
915#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
916 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
917 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
918 pVM->hm.s.vmx.HCPhys##a_Name = 0;
919
920#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
921 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
922 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
923 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
924
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
927#endif
928 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
929
930 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
931 for (VMCPUID i = 0; i < pVM->cCpus; i++)
932 {
933 PVMCPU pVCpu = &pVM->aCpus[i];
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
935 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
936 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
937 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
938 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
939 }
940#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
941#undef VMXLOCAL_INIT_VM_MEMOBJ
942
943 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
944 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
945 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
946 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
947
948 /*
949 * Allocate all the VT-x structures.
950 */
951 int rc = VINF_SUCCESS;
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
957 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
958#endif
959
960 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
961 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
962 {
963 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
964 &pVM->hm.s.vmx.HCPhysApicAccess);
965 if (RT_FAILURE(rc))
966 goto cleanup;
967 }
968
969 /*
970 * Initialize per-VCPU VT-x structures.
971 */
972 for (VMCPUID i = 0; i < pVM->cCpus; i++)
973 {
974 PVMCPU pVCpu = &pVM->aCpus[i];
975 AssertPtr(pVCpu);
976
977 /* Allocate the VM control structure (VMCS). */
978 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
979 if (RT_FAILURE(rc))
980 goto cleanup;
981
982 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
983 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
984 {
985 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
986 &pVCpu->hm.s.vmx.HCPhysVirtApic);
987 if (RT_FAILURE(rc))
988 goto cleanup;
989 }
990
991 /*
992 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
993 * transparent accesses of specific MSRs.
994 *
995 * If the condition for enabling MSR bitmaps changes here, don't forget to
996 * update HMAreMsrBitmapsAvailable().
997 */
998 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
999 {
1000 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1001 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1002 if (RT_FAILURE(rc))
1003 goto cleanup;
1004 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1005 }
1006
1007 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1008 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1009 if (RT_FAILURE(rc))
1010 goto cleanup;
1011
1012 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1013 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016 }
1017
1018 return VINF_SUCCESS;
1019
1020cleanup:
1021 hmR0VmxStructsFree(pVM);
1022 return rc;
1023}
1024
1025
1026/**
1027 * Does global VT-x initialization (called during module initialization).
1028 *
1029 * @returns VBox status code.
1030 */
1031VMMR0DECL(int) VMXR0GlobalInit(void)
1032{
1033#ifdef HMVMX_USE_FUNCTION_TABLE
1034 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1035# ifdef VBOX_STRICT
1036 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1037 Assert(g_apfnVMExitHandlers[i]);
1038# endif
1039#endif
1040 return VINF_SUCCESS;
1041}
1042
1043
1044/**
1045 * Does global VT-x termination (called during module termination).
1046 */
1047VMMR0DECL(void) VMXR0GlobalTerm()
1048{
1049 /* Nothing to do currently. */
1050}
1051
1052
1053/**
1054 * Sets up and activates VT-x on the current CPU.
1055 *
1056 * @returns VBox status code.
1057 * @param pCpu Pointer to the global CPU info struct.
1058 * @param pVM The cross context VM structure. Can be
1059 * NULL after a host resume operation.
1060 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1061 * fEnabledByHost is @c true).
1062 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1063 * @a fEnabledByHost is @c true).
1064 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1065 * enable VT-x on the host.
1066 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1067 */
1068VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1069 void *pvMsrs)
1070{
1071 Assert(pCpu);
1072 Assert(pvMsrs);
1073 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1074
1075 /* Enable VT-x if it's not already enabled by the host. */
1076 if (!fEnabledByHost)
1077 {
1078 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1079 if (RT_FAILURE(rc))
1080 return rc;
1081 }
1082
1083 /*
1084 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1085 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1086 */
1087 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1088 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1089 {
1090 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1091 pCpu->fFlushAsidBeforeUse = false;
1092 }
1093 else
1094 pCpu->fFlushAsidBeforeUse = true;
1095
1096 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1097 ++pCpu->cTlbFlushes;
1098
1099 return VINF_SUCCESS;
1100}
1101
1102
1103/**
1104 * Deactivates VT-x on the current CPU.
1105 *
1106 * @returns VBox status code.
1107 * @param pCpu Pointer to the global CPU info struct.
1108 * @param pvCpuPage Pointer to the VMXON region.
1109 * @param HCPhysCpuPage Physical address of the VMXON region.
1110 *
1111 * @remarks This function should never be called when SUPR0EnableVTx() or
1112 * similar was used to enable VT-x on the host.
1113 */
1114VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1115{
1116 NOREF(pCpu);
1117 NOREF(pvCpuPage);
1118 NOREF(HCPhysCpuPage);
1119
1120 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1121 return hmR0VmxLeaveRootMode();
1122}
1123
1124
1125/**
1126 * Sets the permission bits for the specified MSR in the MSR bitmap.
1127 *
1128 * @param pVCpu The cross context virtual CPU structure.
1129 * @param uMsr The MSR value.
1130 * @param enmRead Whether reading this MSR causes a VM-exit.
1131 * @param enmWrite Whether writing this MSR causes a VM-exit.
1132 */
1133static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1134{
1135 int32_t iBit;
1136 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1137
1138 /*
1139 * Layout:
1140 * 0x000 - 0x3ff - Low MSR read bits
1141 * 0x400 - 0x7ff - High MSR read bits
1142 * 0x800 - 0xbff - Low MSR write bits
1143 * 0xc00 - 0xfff - High MSR write bits
1144 */
1145 if (uMsr <= 0x00001FFF)
1146 iBit = uMsr;
1147 else if ( uMsr >= 0xC0000000
1148 && uMsr <= 0xC0001FFF)
1149 {
1150 iBit = (uMsr - 0xC0000000);
1151 pbMsrBitmap += 0x400;
1152 }
1153 else
1154 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1155
1156 Assert(iBit <= 0x1fff);
1157 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1158 ASMBitSet(pbMsrBitmap, iBit);
1159 else
1160 ASMBitClear(pbMsrBitmap, iBit);
1161
1162 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1163 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1164 else
1165 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1166}
1167
1168
1169#ifdef VBOX_STRICT
1170/**
1171 * Gets the permission bits for the specified MSR in the MSR bitmap.
1172 *
1173 * @returns VBox status code.
1174 * @retval VINF_SUCCESS if the specified MSR is found.
1175 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1176 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1177 *
1178 * @param pVCpu The cross context virtual CPU structure.
1179 * @param uMsr The MSR.
1180 * @param penmRead Where to store the read permissions.
1181 * @param penmWrite Where to store the write permissions.
1182 */
1183static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1184{
1185 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1186 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1187 int32_t iBit;
1188 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1189
1190 /* See hmR0VmxSetMsrPermission() for the layout. */
1191 if (uMsr <= 0x00001FFF)
1192 iBit = uMsr;
1193 else if ( uMsr >= 0xC0000000
1194 && uMsr <= 0xC0001FFF)
1195 {
1196 iBit = (uMsr - 0xC0000000);
1197 pbMsrBitmap += 0x400;
1198 }
1199 else
1200 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1201
1202 Assert(iBit <= 0x1fff);
1203 if (ASMBitTest(pbMsrBitmap, iBit))
1204 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1205 else
1206 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1207
1208 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1209 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1210 else
1211 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1212 return VINF_SUCCESS;
1213}
1214#endif /* VBOX_STRICT */
1215
1216
1217/**
1218 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1219 * area.
1220 *
1221 * @returns VBox status code.
1222 * @param pVCpu The cross context virtual CPU structure.
1223 * @param cMsrs The number of MSRs.
1224 */
1225DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1226{
1227 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1228 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1229 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1230 {
1231 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1232 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1233 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1234 }
1235
1236 /* Update number of guest MSRs to load/store across the world-switch. */
1237 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1238 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1239
1240 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1241 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1242
1243 /* Update the VCPU's copy of the MSR count. */
1244 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1245
1246 return VINF_SUCCESS;
1247}
1248
1249
1250/**
1251 * Adds a new (or updates the value of an existing) guest/host MSR
1252 * pair to be swapped during the world-switch as part of the
1253 * auto-load/store MSR area in the VMCS.
1254 *
1255 * @returns VBox status code.
1256 * @param pVCpu The cross context virtual CPU structure.
1257 * @param uMsr The MSR.
1258 * @param uGuestMsrValue Value of the guest MSR.
1259 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1260 * necessary.
1261 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1262 * its value was updated. Optional, can be NULL.
1263 */
1264static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1265 bool *pfAddedAndUpdated)
1266{
1267 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1268 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1269 uint32_t i;
1270 for (i = 0; i < cMsrs; i++)
1271 {
1272 if (pGuestMsr->u32Msr == uMsr)
1273 break;
1274 pGuestMsr++;
1275 }
1276
1277 bool fAdded = false;
1278 if (i == cMsrs)
1279 {
1280 ++cMsrs;
1281 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1282 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1283
1284 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1285 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1286 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1287
1288 fAdded = true;
1289 }
1290
1291 /* Update the MSR values in the auto-load/store MSR area. */
1292 pGuestMsr->u32Msr = uMsr;
1293 pGuestMsr->u64Value = uGuestMsrValue;
1294
1295 /* Create/update the MSR slot in the host MSR area. */
1296 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1297 pHostMsr += i;
1298 pHostMsr->u32Msr = uMsr;
1299
1300 /*
1301 * Update the host MSR only when requested by the caller AND when we're
1302 * adding it to the auto-load/store area. Otherwise, it would have been
1303 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1304 */
1305 bool fUpdatedMsrValue = false;
1306 if ( fAdded
1307 && fUpdateHostMsr)
1308 {
1309 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1310 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1311 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1312 fUpdatedMsrValue = true;
1313 }
1314
1315 if (pfAddedAndUpdated)
1316 *pfAddedAndUpdated = fUpdatedMsrValue;
1317 return VINF_SUCCESS;
1318}
1319
1320
1321/**
1322 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1323 * auto-load/store MSR area in the VMCS.
1324 *
1325 * @returns VBox status code.
1326 * @param pVCpu The cross context virtual CPU structure.
1327 * @param uMsr The MSR.
1328 */
1329static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1330{
1331 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1332 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1333 for (uint32_t i = 0; i < cMsrs; i++)
1334 {
1335 /* Find the MSR. */
1336 if (pGuestMsr->u32Msr == uMsr)
1337 {
1338 /* If it's the last MSR, simply reduce the count. */
1339 if (i == cMsrs - 1)
1340 {
1341 --cMsrs;
1342 break;
1343 }
1344
1345 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1346 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1347 pLastGuestMsr += cMsrs - 1;
1348 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1349 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1350
1351 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1352 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1353 pLastHostMsr += cMsrs - 1;
1354 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1355 pHostMsr->u64Value = pLastHostMsr->u64Value;
1356 --cMsrs;
1357 break;
1358 }
1359 pGuestMsr++;
1360 }
1361
1362 /* Update the VMCS if the count changed (meaning the MSR was found). */
1363 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1364 {
1365 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1366 AssertRCReturn(rc, rc);
1367
1368 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1369 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1370 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1371
1372 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1373 return VINF_SUCCESS;
1374 }
1375
1376 return VERR_NOT_FOUND;
1377}
1378
1379
1380/**
1381 * Checks if the specified guest MSR is part of the auto-load/store area in
1382 * the VMCS.
1383 *
1384 * @returns true if found, false otherwise.
1385 * @param pVCpu The cross context virtual CPU structure.
1386 * @param uMsr The MSR to find.
1387 */
1388static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1389{
1390 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1391 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1392
1393 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1394 {
1395 if (pGuestMsr->u32Msr == uMsr)
1396 return true;
1397 }
1398 return false;
1399}
1400
1401
1402/**
1403 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1404 *
1405 * @param pVCpu The cross context virtual CPU structure.
1406 *
1407 * @remarks No-long-jump zone!!!
1408 */
1409static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1410{
1411 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1412 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1413 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1414 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1415
1416 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1417 {
1418 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1419
1420 /*
1421 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1422 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1423 */
1424 if (pHostMsr->u32Msr == MSR_K6_EFER)
1425 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1426 else
1427 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1428 }
1429
1430 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1431}
1432
1433
1434#if HC_ARCH_BITS == 64
1435/**
1436 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1437 * perform lazy restoration of the host MSRs while leaving VT-x.
1438 *
1439 * @param pVCpu The cross context virtual CPU structure.
1440 *
1441 * @remarks No-long-jump zone!!!
1442 */
1443static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1444{
1445 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1446
1447 /*
1448 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1449 */
1450 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1451 {
1452 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1453 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1454 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1455 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1456 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1457 }
1458}
1459
1460
1461/**
1462 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1463 * lazily while leaving VT-x.
1464 *
1465 * @returns true if it does, false otherwise.
1466 * @param pVCpu The cross context virtual CPU structure.
1467 * @param uMsr The MSR to check.
1468 */
1469static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1470{
1471 NOREF(pVCpu);
1472 switch (uMsr)
1473 {
1474 case MSR_K8_LSTAR:
1475 case MSR_K6_STAR:
1476 case MSR_K8_SF_MASK:
1477 case MSR_K8_KERNEL_GS_BASE:
1478 return true;
1479 }
1480 return false;
1481}
1482
1483
1484/**
1485 * Saves a set of guest MSRs back into the guest-CPU context.
1486 *
1487 * @param pVCpu The cross context virtual CPU structure.
1488 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1489 * out-of-sync. Make sure to update the required fields
1490 * before using them.
1491 *
1492 * @remarks No-long-jump zone!!!
1493 */
1494static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1495{
1496 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1497 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1498
1499 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1500 {
1501 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1502 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1503 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1504 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1505 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1506 }
1507}
1508
1509
1510/**
1511 * Loads a set of guests MSRs to allow read/passthru to the guest.
1512 *
1513 * The name of this function is slightly confusing. This function does NOT
1514 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1515 * common prefix for functions dealing with "lazy restoration" of the shared
1516 * MSRs.
1517 *
1518 * @param pVCpu The cross context virtual CPU structure.
1519 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1520 * out-of-sync. Make sure to update the required fields
1521 * before using them.
1522 *
1523 * @remarks No-long-jump zone!!!
1524 */
1525static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1526{
1527 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1528 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1529
1530#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1531 do { \
1532 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1533 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1534 else \
1535 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1536 } while (0)
1537
1538 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1539 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1540 {
1541 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1542 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1543 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1544 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1545 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1546 }
1547 else
1548 {
1549 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1550 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1551 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1552 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1553 }
1554
1555#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1556}
1557
1558
1559/**
1560 * Performs lazy restoration of the set of host MSRs if they were previously
1561 * loaded with guest MSR values.
1562 *
1563 * @param pVCpu The cross context virtual CPU structure.
1564 *
1565 * @remarks No-long-jump zone!!!
1566 * @remarks The guest MSRs should have been saved back into the guest-CPU
1567 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1568 */
1569static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1570{
1571 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1572 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1573
1574 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1575 {
1576 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1577 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1578 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1579 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1580 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1581 }
1582 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1583}
1584#endif /* HC_ARCH_BITS == 64 */
1585
1586
1587/**
1588 * Verifies that our cached values of the VMCS controls are all
1589 * consistent with what's actually present in the VMCS.
1590 *
1591 * @returns VBox status code.
1592 * @param pVCpu The cross context virtual CPU structure.
1593 */
1594static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1595{
1596 uint32_t u32Val;
1597 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1598 AssertRCReturn(rc, rc);
1599 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1600 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1601
1602 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1603 AssertRCReturn(rc, rc);
1604 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1605 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1606
1607 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1608 AssertRCReturn(rc, rc);
1609 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1610 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1611
1612 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1613 AssertRCReturn(rc, rc);
1614 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1615 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1616
1617 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1618 {
1619 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1620 AssertRCReturn(rc, rc);
1621 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1622 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1623 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1624 }
1625
1626 return VINF_SUCCESS;
1627}
1628
1629
1630#ifdef VBOX_STRICT
1631/**
1632 * Verifies that our cached host EFER value has not changed
1633 * since we cached it.
1634 *
1635 * @param pVCpu The cross context virtual CPU structure.
1636 */
1637static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1638{
1639 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1640
1641 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1642 {
1643 uint64_t u64Val;
1644 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1645 AssertRC(rc);
1646
1647 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1648 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1649 }
1650}
1651
1652
1653/**
1654 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1655 * VMCS are correct.
1656 *
1657 * @param pVCpu The cross context virtual CPU structure.
1658 */
1659static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1660{
1661 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1662
1663 /* Verify MSR counts in the VMCS are what we think it should be. */
1664 uint32_t cMsrs;
1665 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1666 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1667
1668 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1669 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1670
1671 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1672 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1673
1674 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1675 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1676 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1677 {
1678 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1679 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1680 pGuestMsr->u32Msr, cMsrs));
1681
1682 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1683 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1684 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1685
1686 /* Verify that the permissions are as expected in the MSR bitmap. */
1687 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1688 {
1689 VMXMSREXITREAD enmRead;
1690 VMXMSREXITWRITE enmWrite;
1691 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1692 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1693 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1694 {
1695 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1696 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1697 }
1698 else
1699 {
1700 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1701 pGuestMsr->u32Msr, cMsrs));
1702 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1703 pGuestMsr->u32Msr, cMsrs));
1704 }
1705 }
1706 }
1707}
1708#endif /* VBOX_STRICT */
1709
1710
1711/**
1712 * Flushes the TLB using EPT.
1713 *
1714 * @returns VBox status code.
1715 * @param pVCpu The cross context virtual CPU structure of the calling
1716 * EMT. Can be NULL depending on @a enmFlush.
1717 * @param enmFlush Type of flush.
1718 *
1719 * @remarks Caller is responsible for making sure this function is called only
1720 * when NestedPaging is supported and providing @a enmFlush that is
1721 * supported by the CPU.
1722 * @remarks Can be called with interrupts disabled.
1723 */
1724static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1725{
1726 uint64_t au64Descriptor[2];
1727 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1728 au64Descriptor[0] = 0;
1729 else
1730 {
1731 Assert(pVCpu);
1732 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1733 }
1734 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1735
1736 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1737 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1738 rc));
1739 if ( RT_SUCCESS(rc)
1740 && pVCpu)
1741 {
1742 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1743 }
1744}
1745
1746
1747/**
1748 * Flushes the TLB using VPID.
1749 *
1750 * @returns VBox status code.
1751 * @param pVM The cross context VM structure.
1752 * @param pVCpu The cross context virtual CPU structure of the calling
1753 * EMT. Can be NULL depending on @a enmFlush.
1754 * @param enmFlush Type of flush.
1755 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1756 * on @a enmFlush).
1757 *
1758 * @remarks Can be called with interrupts disabled.
1759 */
1760static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1761{
1762 NOREF(pVM);
1763 AssertPtr(pVM);
1764 Assert(pVM->hm.s.vmx.fVpid);
1765
1766 uint64_t au64Descriptor[2];
1767 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1768 {
1769 au64Descriptor[0] = 0;
1770 au64Descriptor[1] = 0;
1771 }
1772 else
1773 {
1774 AssertPtr(pVCpu);
1775 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1776 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1777 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1778 au64Descriptor[1] = GCPtr;
1779 }
1780
1781 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1782 AssertMsg(rc == VINF_SUCCESS,
1783 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1784 if ( RT_SUCCESS(rc)
1785 && pVCpu)
1786 {
1787 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1788 }
1789}
1790
1791
1792/**
1793 * Invalidates a guest page by guest virtual address. Only relevant for
1794 * EPT/VPID, otherwise there is nothing really to invalidate.
1795 *
1796 * @returns VBox status code.
1797 * @param pVM The cross context VM structure.
1798 * @param pVCpu The cross context virtual CPU structure.
1799 * @param GCVirt Guest virtual address of the page to invalidate.
1800 */
1801VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1802{
1803 AssertPtr(pVM);
1804 AssertPtr(pVCpu);
1805 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1806
1807 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1808 if (!fFlushPending)
1809 {
1810 /*
1811 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1812 * See @bugref{6043} and @bugref{6177}.
1813 *
1814 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1815 * function maybe called in a loop with individual addresses.
1816 */
1817 if (pVM->hm.s.vmx.fVpid)
1818 {
1819 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1820 {
1821 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1822 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1823 }
1824 else
1825 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1826 }
1827 else if (pVM->hm.s.fNestedPaging)
1828 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1829 }
1830
1831 return VINF_SUCCESS;
1832}
1833
1834
1835/**
1836 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1837 * otherwise there is nothing really to invalidate.
1838 *
1839 * @returns VBox status code.
1840 * @param pVM The cross context VM structure.
1841 * @param pVCpu The cross context virtual CPU structure.
1842 * @param GCPhys Guest physical address of the page to invalidate.
1843 */
1844VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1845{
1846 NOREF(pVM); NOREF(GCPhys);
1847 LogFlowFunc(("%RGp\n", GCPhys));
1848
1849 /*
1850 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1851 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1852 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1853 */
1854 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1855 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1856 return VINF_SUCCESS;
1857}
1858
1859
1860/**
1861 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1862 * case where neither EPT nor VPID is supported by the CPU.
1863 *
1864 * @param pVM The cross context VM structure.
1865 * @param pVCpu The cross context virtual CPU structure.
1866 * @param pCpu Pointer to the global HM struct.
1867 *
1868 * @remarks Called with interrupts disabled.
1869 */
1870static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1871{
1872 AssertPtr(pVCpu);
1873 AssertPtr(pCpu);
1874 NOREF(pVM);
1875
1876 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1877
1878 Assert(pCpu->idCpu != NIL_RTCPUID);
1879 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1880 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1881 pVCpu->hm.s.fForceTLBFlush = false;
1882 return;
1883}
1884
1885
1886/**
1887 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1888 *
1889 * @param pVM The cross context VM structure.
1890 * @param pVCpu The cross context virtual CPU structure.
1891 * @param pCpu Pointer to the global HM CPU struct.
1892 * @remarks All references to "ASID" in this function pertains to "VPID" in
1893 * Intel's nomenclature. The reason is, to avoid confusion in compare
1894 * statements since the host-CPU copies are named "ASID".
1895 *
1896 * @remarks Called with interrupts disabled.
1897 */
1898static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1899{
1900#ifdef VBOX_WITH_STATISTICS
1901 bool fTlbFlushed = false;
1902# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1903# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1904 if (!fTlbFlushed) \
1905 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1906 } while (0)
1907#else
1908# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1909# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1910#endif
1911
1912 AssertPtr(pVM);
1913 AssertPtr(pCpu);
1914 AssertPtr(pVCpu);
1915 Assert(pCpu->idCpu != NIL_RTCPUID);
1916
1917 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1918 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1919 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1920
1921 /*
1922 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1923 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1924 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1925 */
1926 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1927 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1928 {
1929 ++pCpu->uCurrentAsid;
1930 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1931 {
1932 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1933 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1934 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1935 }
1936
1937 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1938 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1939 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1940
1941 /*
1942 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1943 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1944 */
1945 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1946 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1947 HMVMX_SET_TAGGED_TLB_FLUSHED();
1948 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1949 }
1950
1951 /* Check for explicit TLB flushes. */
1952 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1953 {
1954 /*
1955 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1956 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1957 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1958 * but not guest-physical mappings.
1959 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1960 */
1961 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1962 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1963 HMVMX_SET_TAGGED_TLB_FLUSHED();
1964 }
1965
1966 pVCpu->hm.s.fForceTLBFlush = false;
1967 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1968
1969 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1970 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1971 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1972 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1973 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1974 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1975 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1976 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1977 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1978
1979 /* Update VMCS with the VPID. */
1980 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1981 AssertRC(rc);
1982
1983#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1984}
1985
1986
1987/**
1988 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1989 *
1990 * @returns VBox status code.
1991 * @param pVM The cross context VM structure.
1992 * @param pVCpu The cross context virtual CPU structure.
1993 * @param pCpu Pointer to the global HM CPU struct.
1994 *
1995 * @remarks Called with interrupts disabled.
1996 */
1997static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1998{
1999 AssertPtr(pVM);
2000 AssertPtr(pVCpu);
2001 AssertPtr(pCpu);
2002 Assert(pCpu->idCpu != NIL_RTCPUID);
2003 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2004 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2005
2006 /*
2007 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2008 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2009 */
2010 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2011 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2012 {
2013 pVCpu->hm.s.fForceTLBFlush = true;
2014 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2015 }
2016
2017 /* Check for explicit TLB flushes. */
2018 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2019 {
2020 pVCpu->hm.s.fForceTLBFlush = true;
2021 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2022 }
2023
2024 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2025 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2026
2027 if (pVCpu->hm.s.fForceTLBFlush)
2028 {
2029 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2030 pVCpu->hm.s.fForceTLBFlush = false;
2031 }
2032}
2033
2034
2035/**
2036 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2037 *
2038 * @returns VBox status code.
2039 * @param pVM The cross context VM structure.
2040 * @param pVCpu The cross context virtual CPU structure.
2041 * @param pCpu Pointer to the global HM CPU struct.
2042 *
2043 * @remarks Called with interrupts disabled.
2044 */
2045static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2046{
2047 AssertPtr(pVM);
2048 AssertPtr(pVCpu);
2049 AssertPtr(pCpu);
2050 Assert(pCpu->idCpu != NIL_RTCPUID);
2051 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2052 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2053
2054 /*
2055 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2056 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2057 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2058 */
2059 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2060 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2061 {
2062 pVCpu->hm.s.fForceTLBFlush = true;
2063 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2064 }
2065
2066 /* Check for explicit TLB flushes. */
2067 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2068 {
2069 /*
2070 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2071 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2072 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2073 */
2074 pVCpu->hm.s.fForceTLBFlush = true;
2075 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2076 }
2077
2078 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2079 if (pVCpu->hm.s.fForceTLBFlush)
2080 {
2081 ++pCpu->uCurrentAsid;
2082 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2083 {
2084 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2085 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2086 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2087 }
2088
2089 pVCpu->hm.s.fForceTLBFlush = false;
2090 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2091 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2092 if (pCpu->fFlushAsidBeforeUse)
2093 {
2094 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2095 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2096 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2097 {
2098 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2099 pCpu->fFlushAsidBeforeUse = false;
2100 }
2101 else
2102 {
2103 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2104 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2105 }
2106 }
2107 }
2108
2109 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2110 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2111 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2112 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2113 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2114 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2115 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2116
2117 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2118 AssertRC(rc);
2119}
2120
2121
2122/**
2123 * Flushes the guest TLB entry based on CPU capabilities.
2124 *
2125 * @param pVCpu The cross context virtual CPU structure.
2126 * @param pCpu Pointer to the global HM CPU struct.
2127 */
2128DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2129{
2130#ifdef HMVMX_ALWAYS_FLUSH_TLB
2131 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2132#endif
2133 PVM pVM = pVCpu->CTX_SUFF(pVM);
2134 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2135 {
2136 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2137 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2138 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2139 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2140 default:
2141 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2142 break;
2143 }
2144
2145 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2146}
2147
2148
2149/**
2150 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2151 * TLB entries from the host TLB before VM-entry.
2152 *
2153 * @returns VBox status code.
2154 * @param pVM The cross context VM structure.
2155 */
2156static int hmR0VmxSetupTaggedTlb(PVM pVM)
2157{
2158 /*
2159 * Determine optimal flush type for Nested Paging.
2160 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2161 * guest execution (see hmR3InitFinalizeR0()).
2162 */
2163 if (pVM->hm.s.fNestedPaging)
2164 {
2165 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2166 {
2167 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2168 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2169 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2170 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2171 else
2172 {
2173 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2174 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2175 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2176 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2177 }
2178
2179 /* Make sure the write-back cacheable memory type for EPT is supported. */
2180 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2181 {
2182 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2183 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2184 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2185 }
2186
2187 /* EPT requires a page-walk length of 4. */
2188 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2189 {
2190 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2191 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2192 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2193 }
2194 }
2195 else
2196 {
2197 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2198 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2199 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2200 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2201 }
2202 }
2203
2204 /*
2205 * Determine optimal flush type for VPID.
2206 */
2207 if (pVM->hm.s.vmx.fVpid)
2208 {
2209 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2210 {
2211 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2212 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2213 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2214 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2215 else
2216 {
2217 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2218 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2219 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2220 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2221 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2222 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2223 pVM->hm.s.vmx.fVpid = false;
2224 }
2225 }
2226 else
2227 {
2228 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2229 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2230 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2231 pVM->hm.s.vmx.fVpid = false;
2232 }
2233 }
2234
2235 /*
2236 * Setup the handler for flushing tagged-TLBs.
2237 */
2238 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2239 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2240 else if (pVM->hm.s.fNestedPaging)
2241 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2242 else if (pVM->hm.s.vmx.fVpid)
2243 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2244 else
2245 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2246 return VINF_SUCCESS;
2247}
2248
2249
2250/**
2251 * Sets up pin-based VM-execution controls in the VMCS.
2252 *
2253 * @returns VBox status code.
2254 * @param pVM The cross context VM structure.
2255 * @param pVCpu The cross context virtual CPU structure.
2256 */
2257static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2258{
2259 AssertPtr(pVM);
2260 AssertPtr(pVCpu);
2261
2262 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2263 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2264
2265 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2266 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2267
2268 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2269 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2270
2271 /* Enable the VMX preemption timer. */
2272 if (pVM->hm.s.vmx.fUsePreemptTimer)
2273 {
2274 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2275 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2276 }
2277
2278 if ((val & zap) != val)
2279 {
2280 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2281 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2282 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2283 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2284 }
2285
2286 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2287 AssertRCReturn(rc, rc);
2288
2289 pVCpu->hm.s.vmx.u32PinCtls = val;
2290 return rc;
2291}
2292
2293
2294/**
2295 * Sets up processor-based VM-execution controls in the VMCS.
2296 *
2297 * @returns VBox status code.
2298 * @param pVM The cross context VM structure.
2299 * @param pVCpu The cross context virtual CPU structure.
2300 */
2301static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2302{
2303 AssertPtr(pVM);
2304 AssertPtr(pVCpu);
2305
2306 int rc = VERR_INTERNAL_ERROR_5;
2307 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2308 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2309
2310 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2311 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2312 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2313 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2314 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2315 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2316 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2317
2318 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2319 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2320 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2321 {
2322 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2323 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2324 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2325 }
2326
2327 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2328 if (!pVM->hm.s.fNestedPaging)
2329 {
2330 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2331 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2332 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2333 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2334 }
2335
2336 /* Use TPR shadowing if supported by the CPU. */
2337 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2338 {
2339 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2340 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2341 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2342 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2343 AssertRCReturn(rc, rc);
2344
2345 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2346 /* CR8 writes cause a VM-exit based on TPR threshold. */
2347 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2348 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2349 }
2350 else
2351 {
2352 /*
2353 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2354 * Set this control only for 64-bit guests.
2355 */
2356 if (pVM->hm.s.fAllow64BitGuests)
2357 {
2358 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2359 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2360 }
2361 }
2362
2363 /* Use MSR-bitmaps if supported by the CPU. */
2364 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2365 {
2366 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2367
2368 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2369 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2370 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2371 AssertRCReturn(rc, rc);
2372
2373 /*
2374 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2375 * automatically using dedicated fields in the VMCS.
2376 */
2377 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2378 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2379 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2380 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2381 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2382
2383#if HC_ARCH_BITS == 64
2384 /*
2385 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2386 */
2387 if (pVM->hm.s.fAllow64BitGuests)
2388 {
2389 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2390 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2391 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2392 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2393 }
2394#endif
2395 }
2396
2397 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2398 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2399 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2400
2401 if ((val & zap) != val)
2402 {
2403 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2404 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2405 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2406 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2407 }
2408
2409 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2410 AssertRCReturn(rc, rc);
2411
2412 pVCpu->hm.s.vmx.u32ProcCtls = val;
2413
2414 /*
2415 * Secondary processor-based VM-execution controls.
2416 */
2417 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2418 {
2419 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2420 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2421
2422 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2423 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2424
2425 if (pVM->hm.s.fNestedPaging)
2426 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2427 else
2428 {
2429 /*
2430 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2431 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2432 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2433 */
2434 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2435 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2436 }
2437
2438 if (pVM->hm.s.vmx.fVpid)
2439 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2440
2441 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2442 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2443
2444 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2445 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2446 * done dynamically. */
2447 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2448 {
2449 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2450 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2451 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2452 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2453 AssertRCReturn(rc, rc);
2454 }
2455
2456 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2457 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2458
2459 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2460 && pVM->hm.s.vmx.cPleGapTicks
2461 && pVM->hm.s.vmx.cPleWindowTicks)
2462 {
2463 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2464
2465 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2466 AssertRCReturn(rc, rc);
2467
2468 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2469 AssertRCReturn(rc, rc);
2470 }
2471
2472 if ((val & zap) != val)
2473 {
2474 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2475 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2476 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2477 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2478 }
2479
2480 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2481 AssertRCReturn(rc, rc);
2482
2483 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2484 }
2485 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2486 {
2487 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2488 "available\n"));
2489 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2490 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2491 }
2492
2493 return VINF_SUCCESS;
2494}
2495
2496
2497/**
2498 * Sets up miscellaneous (everything other than Pin & Processor-based
2499 * VM-execution) control fields in the VMCS.
2500 *
2501 * @returns VBox status code.
2502 * @param pVM The cross context VM structure.
2503 * @param pVCpu The cross context virtual CPU structure.
2504 */
2505static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2506{
2507 NOREF(pVM);
2508 AssertPtr(pVM);
2509 AssertPtr(pVCpu);
2510
2511 int rc = VERR_GENERAL_FAILURE;
2512
2513 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2514#if 0
2515 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2516 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2517 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2518
2519 /*
2520 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2521 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2522 * We thus use the exception bitmap to control it rather than use both.
2523 */
2524 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2525 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2526
2527 /** @todo Explore possibility of using IO-bitmaps. */
2528 /* All IO & IOIO instructions cause VM-exits. */
2529 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2530 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2531
2532 /* Initialize the MSR-bitmap area. */
2533 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2535 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2536#endif
2537
2538 /* Setup MSR auto-load/store area. */
2539 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2540 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2541 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2542 AssertRCReturn(rc, rc);
2543 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2544 AssertRCReturn(rc, rc);
2545
2546 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2547 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2548 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2549 AssertRCReturn(rc, rc);
2550
2551 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2552 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2553 AssertRCReturn(rc, rc);
2554
2555 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2556#if 0
2557 /* Setup debug controls */
2558 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2559 AssertRCReturn(rc, rc);
2560 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2561 AssertRCReturn(rc, rc);
2562#endif
2563
2564 return rc;
2565}
2566
2567
2568/**
2569 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2570 *
2571 * @returns VBox status code.
2572 * @param pVM The cross context VM structure.
2573 * @param pVCpu The cross context virtual CPU structure.
2574 */
2575static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2576{
2577 AssertPtr(pVM);
2578 AssertPtr(pVCpu);
2579
2580 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2581
2582 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2583
2584 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2585 if (!pVM->hm.s.fNestedPaging)
2586 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2587
2588 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2589 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2590 AssertRCReturn(rc, rc);
2591 return rc;
2592}
2593
2594
2595/**
2596 * Sets up the initial guest-state mask. The guest-state mask is consulted
2597 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2598 * for the nested virtualization case (as it would cause a VM-exit).
2599 *
2600 * @param pVCpu The cross context virtual CPU structure.
2601 */
2602static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2603{
2604 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2605 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2606 return VINF_SUCCESS;
2607}
2608
2609
2610/**
2611 * Does per-VM VT-x initialization.
2612 *
2613 * @returns VBox status code.
2614 * @param pVM The cross context VM structure.
2615 */
2616VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2617{
2618 LogFlowFunc(("pVM=%p\n", pVM));
2619
2620 int rc = hmR0VmxStructsAlloc(pVM);
2621 if (RT_FAILURE(rc))
2622 {
2623 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2624 return rc;
2625 }
2626
2627 return VINF_SUCCESS;
2628}
2629
2630
2631/**
2632 * Does per-VM VT-x termination.
2633 *
2634 * @returns VBox status code.
2635 * @param pVM The cross context VM structure.
2636 */
2637VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2638{
2639 LogFlowFunc(("pVM=%p\n", pVM));
2640
2641#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2642 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2643 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2644#endif
2645 hmR0VmxStructsFree(pVM);
2646 return VINF_SUCCESS;
2647}
2648
2649
2650/**
2651 * Sets up the VM for execution under VT-x.
2652 * This function is only called once per-VM during initialization.
2653 *
2654 * @returns VBox status code.
2655 * @param pVM The cross context VM structure.
2656 */
2657VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2658{
2659 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2660 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2661
2662 LogFlowFunc(("pVM=%p\n", pVM));
2663
2664 /*
2665 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2666 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2667 */
2668 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2669 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2670 || !pVM->hm.s.vmx.pRealModeTSS))
2671 {
2672 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2673 return VERR_INTERNAL_ERROR;
2674 }
2675
2676 /* Initialize these always, see hmR3InitFinalizeR0().*/
2677 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2678 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2679
2680 /* Setup the tagged-TLB flush handlers. */
2681 int rc = hmR0VmxSetupTaggedTlb(pVM);
2682 if (RT_FAILURE(rc))
2683 {
2684 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2685 return rc;
2686 }
2687
2688 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2689 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2690#if HC_ARCH_BITS == 64
2691 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2692 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2693 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2694 {
2695 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2696 }
2697#endif
2698
2699 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2700 {
2701 PVMCPU pVCpu = &pVM->aCpus[i];
2702 AssertPtr(pVCpu);
2703 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2704
2705 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2706 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2707
2708 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2709 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2710 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2711
2712 /* Set revision dword at the beginning of the VMCS structure. */
2713 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2714
2715 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2716 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2717 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2718 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2719
2720 /* Load this VMCS as the current VMCS. */
2721 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2722 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2723 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2724
2725 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2726 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2727 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2728
2729 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2730 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2731 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2732
2733 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2734 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2735 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2736
2737 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2738 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2739 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2740
2741 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2742 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2743 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2744
2745#if HC_ARCH_BITS == 32
2746 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2747 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2748 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2749#endif
2750
2751 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2752 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2753 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2754 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2755
2756 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2757
2758 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2759 }
2760
2761 return VINF_SUCCESS;
2762}
2763
2764
2765/**
2766 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2767 * the VMCS.
2768 *
2769 * @returns VBox status code.
2770 * @param pVM The cross context VM structure.
2771 * @param pVCpu The cross context virtual CPU structure.
2772 */
2773DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2774{
2775 NOREF(pVM); NOREF(pVCpu);
2776
2777 RTCCUINTREG uReg = ASMGetCR0();
2778 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2779 AssertRCReturn(rc, rc);
2780
2781 uReg = ASMGetCR3();
2782 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2783 AssertRCReturn(rc, rc);
2784
2785 uReg = ASMGetCR4();
2786 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2787 AssertRCReturn(rc, rc);
2788 return rc;
2789}
2790
2791
2792#if HC_ARCH_BITS == 64
2793/**
2794 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2795 * requirements. See hmR0VmxSaveHostSegmentRegs().
2796 */
2797# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2798 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2799 { \
2800 bool fValidSelector = true; \
2801 if ((selValue) & X86_SEL_LDT) \
2802 { \
2803 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2804 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2805 } \
2806 if (fValidSelector) \
2807 { \
2808 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2809 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2810 } \
2811 (selValue) = 0; \
2812 }
2813#endif
2814
2815
2816/**
2817 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2818 * the host-state area in the VMCS.
2819 *
2820 * @returns VBox status code.
2821 * @param pVM The cross context VM structure.
2822 * @param pVCpu The cross context virtual CPU structure.
2823 */
2824DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2825{
2826 int rc = VERR_INTERNAL_ERROR_5;
2827
2828#if HC_ARCH_BITS == 64
2829 /*
2830 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2831 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2832 */
2833 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2834 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2835#endif
2836
2837 /*
2838 * Host DS, ES, FS and GS segment registers.
2839 */
2840#if HC_ARCH_BITS == 64
2841 RTSEL uSelDS = ASMGetDS();
2842 RTSEL uSelES = ASMGetES();
2843 RTSEL uSelFS = ASMGetFS();
2844 RTSEL uSelGS = ASMGetGS();
2845#else
2846 RTSEL uSelDS = 0;
2847 RTSEL uSelES = 0;
2848 RTSEL uSelFS = 0;
2849 RTSEL uSelGS = 0;
2850#endif
2851
2852 /* Recalculate which host-state bits need to be manually restored. */
2853 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2854
2855 /*
2856 * Host CS and SS segment registers.
2857 */
2858 RTSEL uSelCS = ASMGetCS();
2859 RTSEL uSelSS = ASMGetSS();
2860
2861 /*
2862 * Host TR segment register.
2863 */
2864 RTSEL uSelTR = ASMGetTR();
2865
2866#if HC_ARCH_BITS == 64
2867 /*
2868 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2869 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2870 */
2871 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2872 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2873 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2874 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2875# undef VMXLOCAL_ADJUST_HOST_SEG
2876#endif
2877
2878 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2879 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2880 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2881 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2882 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2883 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2884 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2885 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2886 Assert(uSelCS);
2887 Assert(uSelTR);
2888
2889 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2890#if 0
2891 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2892 Assert(uSelSS != 0);
2893#endif
2894
2895 /* Write these host selector fields into the host-state area in the VMCS. */
2896 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2897 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2898#if HC_ARCH_BITS == 64
2899 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2900 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2901 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2902 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2903#else
2904 NOREF(uSelDS);
2905 NOREF(uSelES);
2906 NOREF(uSelFS);
2907 NOREF(uSelGS);
2908#endif
2909 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2910
2911 /*
2912 * Host GDTR and IDTR.
2913 */
2914 RTGDTR Gdtr;
2915 RTIDTR Idtr;
2916 RT_ZERO(Gdtr);
2917 RT_ZERO(Idtr);
2918 ASMGetGDTR(&Gdtr);
2919 ASMGetIDTR(&Idtr);
2920 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2921 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2922
2923#if HC_ARCH_BITS == 64
2924 /*
2925 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2926 * maximum limit (0xffff) on every VM-exit.
2927 */
2928 if (Gdtr.cbGdt != 0xffff)
2929 {
2930 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2931 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2932 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2933 }
2934
2935 /*
2936 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2937 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2938 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2939 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2940 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
2941 * hosts where we are pretty sure it won't cause trouble.
2942 */
2943# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2944 if (Idtr.cbIdt < 0x0fff)
2945# else
2946 if (Idtr.cbIdt != 0xffff)
2947# endif
2948 {
2949 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2950 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2951 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2952 }
2953#endif
2954
2955 /*
2956 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2957 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2958 */
2959 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
2960 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
2961 VERR_VMX_INVALID_HOST_STATE);
2962
2963 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2964#if HC_ARCH_BITS == 64
2965 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
2966
2967 /*
2968 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2969 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2970 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2971 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2972 *
2973 * [1] See Intel spec. 3.5 "System Descriptor Types".
2974 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2975 */
2976 Assert(pDesc->System.u4Type == 11);
2977 if ( pDesc->System.u16LimitLow != 0x67
2978 || pDesc->System.u4LimitHigh)
2979 {
2980 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2981 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
2982 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
2983 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
2984 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2985
2986 /* Store the GDTR here as we need it while restoring TR. */
2987 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2988 }
2989#else
2990 NOREF(pVM);
2991 uintptr_t uTRBase = X86DESC_BASE(pDesc);
2992#endif
2993 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2994 AssertRCReturn(rc, rc);
2995
2996 /*
2997 * Host FS base and GS base.
2998 */
2999#if HC_ARCH_BITS == 64
3000 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3001 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3002 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3003 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3004
3005 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3006 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3007 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3008 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3009 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3010#endif
3011 return rc;
3012}
3013
3014
3015/**
3016 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3017 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3018 * the host after every successful VM-exit.
3019 *
3020 * @returns VBox status code.
3021 * @param pVM The cross context VM structure.
3022 * @param pVCpu The cross context virtual CPU structure.
3023 *
3024 * @remarks No-long-jump zone!!!
3025 */
3026DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3027{
3028 NOREF(pVM);
3029
3030 AssertPtr(pVCpu);
3031 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3032
3033 int rc = VINF_SUCCESS;
3034#if HC_ARCH_BITS == 64
3035 if (pVM->hm.s.fAllow64BitGuests)
3036 hmR0VmxLazySaveHostMsrs(pVCpu);
3037#endif
3038
3039 /*
3040 * Host Sysenter MSRs.
3041 */
3042 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3043 AssertRCReturn(rc, rc);
3044#if HC_ARCH_BITS == 32
3045 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3046 AssertRCReturn(rc, rc);
3047 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3048#else
3049 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3050 AssertRCReturn(rc, rc);
3051 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3052#endif
3053 AssertRCReturn(rc, rc);
3054
3055 /*
3056 * Host EFER MSR.
3057 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3058 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3059 */
3060 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3061 {
3062 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3063 AssertRCReturn(rc, rc);
3064 }
3065
3066 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3067 * hmR0VmxLoadGuestExitCtls() !! */
3068
3069 return rc;
3070}
3071
3072
3073/**
3074 * Figures out if we need to swap the EFER MSR which is
3075 * particularly expensive.
3076 *
3077 * We check all relevant bits. For now, that's everything
3078 * besides LMA/LME, as these two bits are handled by VM-entry,
3079 * see hmR0VmxLoadGuestExitCtls() and
3080 * hmR0VMxLoadGuestEntryCtls().
3081 *
3082 * @returns true if we need to load guest EFER, false otherwise.
3083 * @param pVCpu The cross context virtual CPU structure.
3084 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3085 * out-of-sync. Make sure to update the required fields
3086 * before using them.
3087 *
3088 * @remarks Requires EFER, CR4.
3089 * @remarks No-long-jump zone!!!
3090 */
3091static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3092{
3093#ifdef HMVMX_ALWAYS_SWAP_EFER
3094 return true;
3095#endif
3096
3097#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3098 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3099 if (CPUMIsGuestInLongMode(pVCpu))
3100 return false;
3101#endif
3102
3103 PVM pVM = pVCpu->CTX_SUFF(pVM);
3104 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3105 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3106
3107 /*
3108 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3109 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3110 */
3111 if ( CPUMIsGuestInLongMode(pVCpu)
3112 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3113 {
3114 return true;
3115 }
3116
3117 /*
3118 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3119 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3120 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3121 */
3122 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3123 && (pMixedCtx->cr0 & X86_CR0_PG)
3124 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3125 {
3126 /* Assert that host is PAE capable. */
3127 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3128 return true;
3129 }
3130
3131 /** @todo Check the latest Intel spec. for any other bits,
3132 * like SMEP/SMAP? */
3133 return false;
3134}
3135
3136
3137/**
3138 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3139 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3140 * controls".
3141 *
3142 * @returns VBox status code.
3143 * @param pVCpu The cross context virtual CPU structure.
3144 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3145 * out-of-sync. Make sure to update the required fields
3146 * before using them.
3147 *
3148 * @remarks Requires EFER.
3149 * @remarks No-long-jump zone!!!
3150 */
3151DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3152{
3153 int rc = VINF_SUCCESS;
3154 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3155 {
3156 PVM pVM = pVCpu->CTX_SUFF(pVM);
3157 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3158 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3159
3160 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3161 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3162
3163 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3164 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3165 {
3166 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3167 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3168 }
3169 else
3170 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3171
3172 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3173 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3174 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3175 {
3176 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3177 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3178 }
3179
3180 /*
3181 * The following should -not- be set (since we're not in SMM mode):
3182 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3183 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3184 */
3185
3186 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3187 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3188
3189 if ((val & zap) != val)
3190 {
3191 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3192 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3193 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3194 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3195 }
3196
3197 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3198 AssertRCReturn(rc, rc);
3199
3200 pVCpu->hm.s.vmx.u32EntryCtls = val;
3201 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3202 }
3203 return rc;
3204}
3205
3206
3207/**
3208 * Sets up the VM-exit controls in the VMCS.
3209 *
3210 * @returns VBox status code.
3211 * @param pVCpu The cross context virtual CPU structure.
3212 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3213 * out-of-sync. Make sure to update the required fields
3214 * before using them.
3215 *
3216 * @remarks Requires EFER.
3217 */
3218DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3219{
3220 NOREF(pMixedCtx);
3221
3222 int rc = VINF_SUCCESS;
3223 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3224 {
3225 PVM pVM = pVCpu->CTX_SUFF(pVM);
3226 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3227 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3228
3229 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3230 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3231
3232 /*
3233 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3234 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3235 */
3236#if HC_ARCH_BITS == 64
3237 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3238 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3239#else
3240 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3241 {
3242 /* The switcher returns to long mode, EFER is managed by the switcher. */
3243 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3244 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3245 }
3246 else
3247 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3248#endif
3249
3250 /* If the newer VMCS fields for managing EFER exists, use it. */
3251 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3252 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3253 {
3254 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3255 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3256 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3257 }
3258
3259 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3260 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3261
3262 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3263 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3264 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3265
3266 if ( pVM->hm.s.vmx.fUsePreemptTimer
3267 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3268 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3269
3270 if ((val & zap) != val)
3271 {
3272 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3273 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3274 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3275 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3276 }
3277
3278 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3279 AssertRCReturn(rc, rc);
3280
3281 pVCpu->hm.s.vmx.u32ExitCtls = val;
3282 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3283 }
3284 return rc;
3285}
3286
3287
3288/**
3289 * Loads the guest APIC and related state.
3290 *
3291 * @returns VBox status code.
3292 * @param pVCpu The cross context virtual CPU structure.
3293 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3294 * out-of-sync. Make sure to update the required fields
3295 * before using them.
3296 */
3297DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3298{
3299 NOREF(pMixedCtx);
3300
3301 int rc = VINF_SUCCESS;
3302 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3303 {
3304 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3305 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3306 {
3307 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3308
3309 bool fPendingIntr = false;
3310 uint8_t u8Tpr = 0;
3311 uint8_t u8PendingIntr = 0;
3312 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3313 AssertRCReturn(rc, rc);
3314
3315 /*
3316 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3317 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3318 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3319 * the interrupt when we VM-exit for other reasons.
3320 */
3321 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3322 uint32_t u32TprThreshold = 0;
3323 if (fPendingIntr)
3324 {
3325 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3326 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3327 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3328 if (u8PendingPriority <= u8TprPriority)
3329 u32TprThreshold = u8PendingPriority;
3330 else
3331 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3332 }
3333 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3334
3335 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3336 AssertRCReturn(rc, rc);
3337 }
3338
3339 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3340 }
3341 return rc;
3342}
3343
3344
3345/**
3346 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3347 *
3348 * @returns Guest's interruptibility-state.
3349 * @param pVCpu The cross context virtual CPU structure.
3350 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3351 * out-of-sync. Make sure to update the required fields
3352 * before using them.
3353 *
3354 * @remarks No-long-jump zone!!!
3355 */
3356DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3357{
3358 /*
3359 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3360 */
3361 uint32_t uIntrState = 0;
3362 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3363 {
3364 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3365 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3366 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3367 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3368 {
3369 if (pMixedCtx->eflags.Bits.u1IF)
3370 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3371 else
3372 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3373 }
3374 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3375 }
3376
3377 /*
3378 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3379 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3380 * setting this would block host-NMIs and IRET will not clear the blocking.
3381 *
3382 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3383 */
3384 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3385 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3386 {
3387 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3388 }
3389
3390 return uIntrState;
3391}
3392
3393
3394/**
3395 * Loads the guest's interruptibility-state into the guest-state area in the
3396 * VMCS.
3397 *
3398 * @returns VBox status code.
3399 * @param pVCpu The cross context virtual CPU structure.
3400 * @param uIntrState The interruptibility-state to set.
3401 */
3402static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3403{
3404 NOREF(pVCpu);
3405 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3406 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3407 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3408 AssertRCReturn(rc, rc);
3409 return rc;
3410}
3411
3412
3413/**
3414 * Loads the exception intercepts required for guest execution in the VMCS.
3415 *
3416 * @returns VBox status code.
3417 * @param pVCpu The cross context virtual CPU structure.
3418 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3419 * out-of-sync. Make sure to update the required fields
3420 * before using them.
3421 */
3422static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3423{
3424 NOREF(pMixedCtx);
3425 int rc = VINF_SUCCESS;
3426 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3427 {
3428 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3429 if (pVCpu->hm.s.fGIMTrapXcptUD)
3430 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3431 else
3432 {
3433#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3434 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3435#endif
3436 }
3437
3438 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3439 AssertRCReturn(rc, rc);
3440
3441 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3442 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3443 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3444 }
3445 return rc;
3446}
3447
3448
3449/**
3450 * Loads the guest's RIP into the guest-state area in the VMCS.
3451 *
3452 * @returns VBox status code.
3453 * @param pVCpu The cross context virtual CPU structure.
3454 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3455 * out-of-sync. Make sure to update the required fields
3456 * before using them.
3457 *
3458 * @remarks No-long-jump zone!!!
3459 */
3460static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3461{
3462 int rc = VINF_SUCCESS;
3463 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3464 {
3465 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3466 AssertRCReturn(rc, rc);
3467
3468 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3469 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3470 HMCPU_CF_VALUE(pVCpu)));
3471 }
3472 return rc;
3473}
3474
3475
3476/**
3477 * Loads the guest's RSP into the guest-state area in the VMCS.
3478 *
3479 * @returns VBox status code.
3480 * @param pVCpu The cross context virtual CPU structure.
3481 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3482 * out-of-sync. Make sure to update the required fields
3483 * before using them.
3484 *
3485 * @remarks No-long-jump zone!!!
3486 */
3487static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3488{
3489 int rc = VINF_SUCCESS;
3490 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3491 {
3492 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3493 AssertRCReturn(rc, rc);
3494
3495 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3496 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3497 }
3498 return rc;
3499}
3500
3501
3502/**
3503 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3504 *
3505 * @returns VBox status code.
3506 * @param pVCpu The cross context virtual CPU structure.
3507 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3508 * out-of-sync. Make sure to update the required fields
3509 * before using them.
3510 *
3511 * @remarks No-long-jump zone!!!
3512 */
3513static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3514{
3515 int rc = VINF_SUCCESS;
3516 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3517 {
3518 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3519 Let us assert it as such and use 32-bit VMWRITE. */
3520 Assert(!(pMixedCtx->rflags.u64 >> 32));
3521 X86EFLAGS Eflags = pMixedCtx->eflags;
3522 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3523 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3524 * These will never be cleared/set, unless some other part of the VMM
3525 * code is buggy - in which case we're better of finding and fixing
3526 * those bugs than hiding them. */
3527 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3528 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3529 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3530 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3531
3532 /*
3533 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3534 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3535 */
3536 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3537 {
3538 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3539 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3540 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3541 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3542 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3543 }
3544
3545 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3546 AssertRCReturn(rc, rc);
3547
3548 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3549 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3550 }
3551 return rc;
3552}
3553
3554
3555/**
3556 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3557 *
3558 * @returns VBox status code.
3559 * @param pVCpu The cross context virtual CPU structure.
3560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3561 * out-of-sync. Make sure to update the required fields
3562 * before using them.
3563 *
3564 * @remarks No-long-jump zone!!!
3565 */
3566DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3567{
3568 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3569 AssertRCReturn(rc, rc);
3570 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3571 AssertRCReturn(rc, rc);
3572 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3573 AssertRCReturn(rc, rc);
3574 return rc;
3575}
3576
3577
3578/**
3579 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3580 * CR0 is partially shared with the host and we have to consider the FPU bits.
3581 *
3582 * @returns VBox status code.
3583 * @param pVCpu The cross context virtual CPU structure.
3584 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3585 * out-of-sync. Make sure to update the required fields
3586 * before using them.
3587 *
3588 * @remarks No-long-jump zone!!!
3589 */
3590static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3591{
3592 /*
3593 * Guest CR0.
3594 * Guest FPU.
3595 */
3596 int rc = VINF_SUCCESS;
3597 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3598 {
3599 Assert(!(pMixedCtx->cr0 >> 32));
3600 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3601 PVM pVM = pVCpu->CTX_SUFF(pVM);
3602
3603 /* The guest's view (read access) of its CR0 is unblemished. */
3604 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3605 AssertRCReturn(rc, rc);
3606 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3607
3608 /* Setup VT-x's view of the guest CR0. */
3609 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3610 if (pVM->hm.s.fNestedPaging)
3611 {
3612 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3613 {
3614 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3615 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3616 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3617 }
3618 else
3619 {
3620 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3621 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3622 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3623 }
3624
3625 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3626 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3627 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3628
3629 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3630 AssertRCReturn(rc, rc);
3631 }
3632 else
3633 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3634
3635 /*
3636 * Guest FPU bits.
3637 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3638 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3639 */
3640 u32GuestCR0 |= X86_CR0_NE;
3641 bool fInterceptNM = false;
3642 if (CPUMIsGuestFPUStateActive(pVCpu))
3643 {
3644 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3645 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3646 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3647 }
3648 else
3649 {
3650 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3651 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3652 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3653 }
3654
3655 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3656 bool fInterceptMF = false;
3657 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3658 fInterceptMF = true;
3659
3660 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3661 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3662 {
3663 Assert(PDMVmmDevHeapIsEnabled(pVM));
3664 Assert(pVM->hm.s.vmx.pRealModeTSS);
3665 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3666 fInterceptNM = true;
3667 fInterceptMF = true;
3668 }
3669 else
3670 {
3671 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3672 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3673 }
3674 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3675
3676 if (fInterceptNM)
3677 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3678 else
3679 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3680
3681 if (fInterceptMF)
3682 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3683 else
3684 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3685
3686 /* Additional intercepts for debugging, define these yourself explicitly. */
3687#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3688 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3689 | RT_BIT(X86_XCPT_BP)
3690 | RT_BIT(X86_XCPT_DB)
3691 | RT_BIT(X86_XCPT_DE)
3692 | RT_BIT(X86_XCPT_NM)
3693 | RT_BIT(X86_XCPT_TS)
3694 | RT_BIT(X86_XCPT_UD)
3695 | RT_BIT(X86_XCPT_NP)
3696 | RT_BIT(X86_XCPT_SS)
3697 | RT_BIT(X86_XCPT_GP)
3698 | RT_BIT(X86_XCPT_PF)
3699 | RT_BIT(X86_XCPT_MF)
3700 ;
3701#elif defined(HMVMX_ALWAYS_TRAP_PF)
3702 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3703#endif
3704
3705 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3706
3707 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3708 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3709 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3710 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3711 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3712 else
3713 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3714
3715 u32GuestCR0 |= uSetCR0;
3716 u32GuestCR0 &= uZapCR0;
3717 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3718
3719 /* Write VT-x's view of the guest CR0 into the VMCS. */
3720 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3721 AssertRCReturn(rc, rc);
3722 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3723 uZapCR0));
3724
3725 /*
3726 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3727 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3728 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3729 */
3730 uint32_t u32CR0Mask = 0;
3731 u32CR0Mask = X86_CR0_PE
3732 | X86_CR0_NE
3733 | X86_CR0_WP
3734 | X86_CR0_PG
3735 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3736 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3737 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3738
3739 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3740 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3741 * and @bugref{6944}. */
3742#if 0
3743 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3744 u32CR0Mask &= ~X86_CR0_PE;
3745#endif
3746 if (pVM->hm.s.fNestedPaging)
3747 u32CR0Mask &= ~X86_CR0_WP;
3748
3749 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3750 if (fInterceptNM)
3751 {
3752 u32CR0Mask |= X86_CR0_TS
3753 | X86_CR0_MP;
3754 }
3755
3756 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3757 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3758 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3759 AssertRCReturn(rc, rc);
3760 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3761
3762 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3763 }
3764 return rc;
3765}
3766
3767
3768/**
3769 * Loads the guest control registers (CR3, CR4) into the guest-state area
3770 * in the VMCS.
3771 *
3772 * @returns VBox status code.
3773 * @param pVCpu The cross context virtual CPU structure.
3774 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3775 * out-of-sync. Make sure to update the required fields
3776 * before using them.
3777 *
3778 * @remarks No-long-jump zone!!!
3779 */
3780static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3781{
3782 int rc = VINF_SUCCESS;
3783 PVM pVM = pVCpu->CTX_SUFF(pVM);
3784
3785 /*
3786 * Guest CR2.
3787 * It's always loaded in the assembler code. Nothing to do here.
3788 */
3789
3790 /*
3791 * Guest CR3.
3792 */
3793 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3794 {
3795 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3796 if (pVM->hm.s.fNestedPaging)
3797 {
3798 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3799
3800 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3801 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3802 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3803 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3804
3805 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3806 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3807 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3808
3809 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3810 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3811 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3812 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3813 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3814 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3815 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3816
3817 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3818 AssertRCReturn(rc, rc);
3819 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3820
3821 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3822 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3823 {
3824 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3825 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3826 {
3827 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3828 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3829 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3830 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3831 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3832 }
3833
3834 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3835 have Unrestricted Execution to handle the guest when it's not using paging. */
3836 GCPhysGuestCR3 = pMixedCtx->cr3;
3837 }
3838 else
3839 {
3840 /*
3841 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3842 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3843 * EPT takes care of translating it to host-physical addresses.
3844 */
3845 RTGCPHYS GCPhys;
3846 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3847 Assert(PDMVmmDevHeapIsEnabled(pVM));
3848
3849 /* We obtain it here every time as the guest could have relocated this PCI region. */
3850 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3851 AssertRCReturn(rc, rc);
3852
3853 GCPhysGuestCR3 = GCPhys;
3854 }
3855
3856 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3857 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3858 }
3859 else
3860 {
3861 /* Non-nested paging case, just use the hypervisor's CR3. */
3862 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3863
3864 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3865 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3866 }
3867 AssertRCReturn(rc, rc);
3868
3869 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3870 }
3871
3872 /*
3873 * Guest CR4.
3874 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3875 */
3876 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3877 {
3878 Assert(!(pMixedCtx->cr4 >> 32));
3879 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3880
3881 /* The guest's view of its CR4 is unblemished. */
3882 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3883 AssertRCReturn(rc, rc);
3884 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3885
3886 /* Setup VT-x's view of the guest CR4. */
3887 /*
3888 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3889 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3890 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3891 */
3892 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3893 {
3894 Assert(pVM->hm.s.vmx.pRealModeTSS);
3895 Assert(PDMVmmDevHeapIsEnabled(pVM));
3896 u32GuestCR4 &= ~X86_CR4_VME;
3897 }
3898
3899 if (pVM->hm.s.fNestedPaging)
3900 {
3901 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3902 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3903 {
3904 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3905 u32GuestCR4 |= X86_CR4_PSE;
3906 /* Our identity mapping is a 32-bit page directory. */
3907 u32GuestCR4 &= ~X86_CR4_PAE;
3908 }
3909 /* else use guest CR4.*/
3910 }
3911 else
3912 {
3913 /*
3914 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3915 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3916 */
3917 switch (pVCpu->hm.s.enmShadowMode)
3918 {
3919 case PGMMODE_REAL: /* Real-mode. */
3920 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3921 case PGMMODE_32_BIT: /* 32-bit paging. */
3922 {
3923 u32GuestCR4 &= ~X86_CR4_PAE;
3924 break;
3925 }
3926
3927 case PGMMODE_PAE: /* PAE paging. */
3928 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3929 {
3930 u32GuestCR4 |= X86_CR4_PAE;
3931 break;
3932 }
3933
3934 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3935 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3936#ifdef VBOX_ENABLE_64_BITS_GUESTS
3937 break;
3938#endif
3939 default:
3940 AssertFailed();
3941 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3942 }
3943 }
3944
3945 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3946 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3947 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3948 u32GuestCR4 |= uSetCR4;
3949 u32GuestCR4 &= uZapCR4;
3950
3951 /* Write VT-x's view of the guest CR4 into the VMCS. */
3952 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
3953 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3954 AssertRCReturn(rc, rc);
3955
3956 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3957 uint32_t u32CR4Mask = X86_CR4_VME
3958 | X86_CR4_PAE
3959 | X86_CR4_PGE
3960 | X86_CR4_PSE
3961 | X86_CR4_VMXE;
3962 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
3963 u32CR4Mask |= X86_CR4_OSXSAVE;
3964 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3965 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3966 AssertRCReturn(rc, rc);
3967
3968 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
3969 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
3970
3971 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3972 }
3973 return rc;
3974}
3975
3976
3977/**
3978 * Loads the guest debug registers into the guest-state area in the VMCS.
3979 *
3980 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
3981 *
3982 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3983 *
3984 * @returns VBox status code.
3985 * @param pVCpu The cross context virtual CPU structure.
3986 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3987 * out-of-sync. Make sure to update the required fields
3988 * before using them.
3989 *
3990 * @remarks No-long-jump zone!!!
3991 */
3992static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3993{
3994 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3995 return VINF_SUCCESS;
3996
3997#ifdef VBOX_STRICT
3998 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3999 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4000 {
4001 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4002 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4003 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4004 }
4005#endif
4006
4007 int rc;
4008 PVM pVM = pVCpu->CTX_SUFF(pVM);
4009 bool fInterceptDB = false;
4010 bool fInterceptMovDRx = false;
4011 if (pVCpu->hm.s.fSingleInstruction)
4012 {
4013 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4014 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4015 {
4016 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4017 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4018 AssertRCReturn(rc, rc);
4019 Assert(fInterceptDB == false);
4020 }
4021 else
4022 {
4023 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4024 pVCpu->hm.s.fClearTrapFlag = true;
4025 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4026 fInterceptDB = true;
4027 }
4028 }
4029
4030 if ( fInterceptDB
4031 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4032 {
4033 /*
4034 * Use the combined guest and host DRx values found in the hypervisor
4035 * register set because the debugger has breakpoints active or someone
4036 * is single stepping on the host side without a monitor trap flag.
4037 *
4038 * Note! DBGF expects a clean DR6 state before executing guest code.
4039 */
4040#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4041 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4042 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4043 {
4044 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4045 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4046 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4047 }
4048 else
4049#endif
4050 if (!CPUMIsHyperDebugStateActive(pVCpu))
4051 {
4052 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4053 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4054 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4055 }
4056
4057 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4058 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4059 AssertRCReturn(rc, rc);
4060
4061 pVCpu->hm.s.fUsingHyperDR7 = true;
4062 fInterceptDB = true;
4063 fInterceptMovDRx = true;
4064 }
4065 else
4066 {
4067 /*
4068 * If the guest has enabled debug registers, we need to load them prior to
4069 * executing guest code so they'll trigger at the right time.
4070 */
4071 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4072 {
4073#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4074 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4075 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4076 {
4077 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4078 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4079 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4080 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4081 }
4082 else
4083#endif
4084 if (!CPUMIsGuestDebugStateActive(pVCpu))
4085 {
4086 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4087 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4088 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4089 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4090 }
4091 Assert(!fInterceptDB);
4092 Assert(!fInterceptMovDRx);
4093 }
4094 /*
4095 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4096 * must intercept #DB in order to maintain a correct DR6 guest value.
4097 */
4098#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4099 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4100 && !CPUMIsGuestDebugStateActive(pVCpu))
4101#else
4102 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4103#endif
4104 {
4105 fInterceptMovDRx = true;
4106 fInterceptDB = true;
4107 }
4108
4109 /* Update guest DR7. */
4110 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4111 AssertRCReturn(rc, rc);
4112
4113 pVCpu->hm.s.fUsingHyperDR7 = false;
4114 }
4115
4116 /*
4117 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4118 */
4119 if ( fInterceptDB
4120 || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4121 {
4122 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4123 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
4124 }
4125 else
4126 {
4127#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4128 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4129 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
4130#endif
4131 }
4132
4133 /*
4134 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4135 */
4136 if (fInterceptMovDRx)
4137 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4138 else
4139 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4140 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4141 AssertRCReturn(rc, rc);
4142
4143 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4144 return VINF_SUCCESS;
4145}
4146
4147
4148#ifdef VBOX_STRICT
4149/**
4150 * Strict function to validate segment registers.
4151 *
4152 * @remarks ASSUMES CR0 is up to date.
4153 */
4154static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4155{
4156 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4157 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4158 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4159 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4160 && ( !CPUMIsGuestInRealModeEx(pCtx)
4161 && !CPUMIsGuestInV86ModeEx(pCtx)))
4162 {
4163 /* Protected mode checks */
4164 /* CS */
4165 Assert(pCtx->cs.Attr.n.u1Present);
4166 Assert(!(pCtx->cs.Attr.u & 0xf00));
4167 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4168 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4169 || !(pCtx->cs.Attr.n.u1Granularity));
4170 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4171 || (pCtx->cs.Attr.n.u1Granularity));
4172 /* CS cannot be loaded with NULL in protected mode. */
4173 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4174 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4175 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4176 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4177 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4178 else
4179 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4180 /* SS */
4181 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4182 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4183 if ( !(pCtx->cr0 & X86_CR0_PE)
4184 || pCtx->cs.Attr.n.u4Type == 3)
4185 {
4186 Assert(!pCtx->ss.Attr.n.u2Dpl);
4187 }
4188 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4189 {
4190 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4191 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4192 Assert(pCtx->ss.Attr.n.u1Present);
4193 Assert(!(pCtx->ss.Attr.u & 0xf00));
4194 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4195 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4196 || !(pCtx->ss.Attr.n.u1Granularity));
4197 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4198 || (pCtx->ss.Attr.n.u1Granularity));
4199 }
4200 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4201 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4202 {
4203 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4204 Assert(pCtx->ds.Attr.n.u1Present);
4205 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4206 Assert(!(pCtx->ds.Attr.u & 0xf00));
4207 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4208 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4209 || !(pCtx->ds.Attr.n.u1Granularity));
4210 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4211 || (pCtx->ds.Attr.n.u1Granularity));
4212 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4213 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4214 }
4215 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4216 {
4217 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4218 Assert(pCtx->es.Attr.n.u1Present);
4219 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4220 Assert(!(pCtx->es.Attr.u & 0xf00));
4221 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4222 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4223 || !(pCtx->es.Attr.n.u1Granularity));
4224 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4225 || (pCtx->es.Attr.n.u1Granularity));
4226 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4227 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4228 }
4229 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4230 {
4231 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4232 Assert(pCtx->fs.Attr.n.u1Present);
4233 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4234 Assert(!(pCtx->fs.Attr.u & 0xf00));
4235 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4236 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4237 || !(pCtx->fs.Attr.n.u1Granularity));
4238 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4239 || (pCtx->fs.Attr.n.u1Granularity));
4240 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4241 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4242 }
4243 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4244 {
4245 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4246 Assert(pCtx->gs.Attr.n.u1Present);
4247 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4248 Assert(!(pCtx->gs.Attr.u & 0xf00));
4249 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4250 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4251 || !(pCtx->gs.Attr.n.u1Granularity));
4252 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4253 || (pCtx->gs.Attr.n.u1Granularity));
4254 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4255 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4256 }
4257 /* 64-bit capable CPUs. */
4258# if HC_ARCH_BITS == 64
4259 Assert(!(pCtx->cs.u64Base >> 32));
4260 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4261 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4262 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4263# endif
4264 }
4265 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4266 || ( CPUMIsGuestInRealModeEx(pCtx)
4267 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4268 {
4269 /* Real and v86 mode checks. */
4270 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4271 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4272 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4273 {
4274 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4275 }
4276 else
4277 {
4278 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4279 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4280 }
4281
4282 /* CS */
4283 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4284 Assert(pCtx->cs.u32Limit == 0xffff);
4285 Assert(u32CSAttr == 0xf3);
4286 /* SS */
4287 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4288 Assert(pCtx->ss.u32Limit == 0xffff);
4289 Assert(u32SSAttr == 0xf3);
4290 /* DS */
4291 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4292 Assert(pCtx->ds.u32Limit == 0xffff);
4293 Assert(u32DSAttr == 0xf3);
4294 /* ES */
4295 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4296 Assert(pCtx->es.u32Limit == 0xffff);
4297 Assert(u32ESAttr == 0xf3);
4298 /* FS */
4299 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4300 Assert(pCtx->fs.u32Limit == 0xffff);
4301 Assert(u32FSAttr == 0xf3);
4302 /* GS */
4303 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4304 Assert(pCtx->gs.u32Limit == 0xffff);
4305 Assert(u32GSAttr == 0xf3);
4306 /* 64-bit capable CPUs. */
4307# if HC_ARCH_BITS == 64
4308 Assert(!(pCtx->cs.u64Base >> 32));
4309 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4310 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4311 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4312# endif
4313 }
4314}
4315#endif /* VBOX_STRICT */
4316
4317
4318/**
4319 * Writes a guest segment register into the guest-state area in the VMCS.
4320 *
4321 * @returns VBox status code.
4322 * @param pVCpu The cross context virtual CPU structure.
4323 * @param idxSel Index of the selector in the VMCS.
4324 * @param idxLimit Index of the segment limit in the VMCS.
4325 * @param idxBase Index of the segment base in the VMCS.
4326 * @param idxAccess Index of the access rights of the segment in the VMCS.
4327 * @param pSelReg Pointer to the segment selector.
4328 *
4329 * @remarks No-long-jump zone!!!
4330 */
4331static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4332 uint32_t idxAccess, PCPUMSELREG pSelReg)
4333{
4334 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4335 AssertRCReturn(rc, rc);
4336 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4337 AssertRCReturn(rc, rc);
4338 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4339 AssertRCReturn(rc, rc);
4340
4341 uint32_t u32Access = pSelReg->Attr.u;
4342 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4343 {
4344 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4345 u32Access = 0xf3;
4346 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4347 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4348 }
4349 else
4350 {
4351 /*
4352 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4353 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4354 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4355 * loaded in protected-mode have their attribute as 0.
4356 */
4357 if (!u32Access)
4358 u32Access = X86DESCATTR_UNUSABLE;
4359 }
4360
4361 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4362 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4363 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4364
4365 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4366 AssertRCReturn(rc, rc);
4367 return rc;
4368}
4369
4370
4371/**
4372 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4373 * into the guest-state area in the VMCS.
4374 *
4375 * @returns VBox status code.
4376 * @param pVCpu The cross context virtual CPU structure.
4377 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4378 * out-of-sync. Make sure to update the required fields
4379 * before using them.
4380 *
4381 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4382 * @remarks No-long-jump zone!!!
4383 */
4384static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4385{
4386 int rc = VERR_INTERNAL_ERROR_5;
4387 PVM pVM = pVCpu->CTX_SUFF(pVM);
4388
4389 /*
4390 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4391 */
4392 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4393 {
4394 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4395 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4396 {
4397 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4398 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4399 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4400 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4401 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4402 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4403 }
4404
4405#ifdef VBOX_WITH_REM
4406 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4407 {
4408 Assert(pVM->hm.s.vmx.pRealModeTSS);
4409 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4410 if ( pVCpu->hm.s.vmx.fWasInRealMode
4411 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4412 {
4413 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4414 in real-mode (e.g. OpenBSD 4.0) */
4415 REMFlushTBs(pVM);
4416 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4417 pVCpu->hm.s.vmx.fWasInRealMode = false;
4418 }
4419 }
4420#endif
4421 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4422 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4423 AssertRCReturn(rc, rc);
4424 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4425 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4426 AssertRCReturn(rc, rc);
4427 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4428 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4429 AssertRCReturn(rc, rc);
4430 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4431 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4432 AssertRCReturn(rc, rc);
4433 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4434 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4435 AssertRCReturn(rc, rc);
4436 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4437 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4438 AssertRCReturn(rc, rc);
4439
4440#ifdef VBOX_STRICT
4441 /* Validate. */
4442 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4443#endif
4444
4445 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4446 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4447 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4448 }
4449
4450 /*
4451 * Guest TR.
4452 */
4453 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4454 {
4455 /*
4456 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4457 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4458 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4459 */
4460 uint16_t u16Sel = 0;
4461 uint32_t u32Limit = 0;
4462 uint64_t u64Base = 0;
4463 uint32_t u32AccessRights = 0;
4464
4465 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4466 {
4467 u16Sel = pMixedCtx->tr.Sel;
4468 u32Limit = pMixedCtx->tr.u32Limit;
4469 u64Base = pMixedCtx->tr.u64Base;
4470 u32AccessRights = pMixedCtx->tr.Attr.u;
4471 }
4472 else
4473 {
4474 Assert(pVM->hm.s.vmx.pRealModeTSS);
4475 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4476
4477 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4478 RTGCPHYS GCPhys;
4479 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4480 AssertRCReturn(rc, rc);
4481
4482 X86DESCATTR DescAttr;
4483 DescAttr.u = 0;
4484 DescAttr.n.u1Present = 1;
4485 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4486
4487 u16Sel = 0;
4488 u32Limit = HM_VTX_TSS_SIZE;
4489 u64Base = GCPhys; /* in real-mode phys = virt. */
4490 u32AccessRights = DescAttr.u;
4491 }
4492
4493 /* Validate. */
4494 Assert(!(u16Sel & RT_BIT(2)));
4495 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4496 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4497 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4498 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4499 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4500 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4501 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4502 Assert( (u32Limit & 0xfff) == 0xfff
4503 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4504 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4505 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4506
4507 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4508 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4509 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4510 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4511
4512 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4513 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4514 }
4515
4516 /*
4517 * Guest GDTR.
4518 */
4519 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4520 {
4521 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4522 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4523
4524 /* Validate. */
4525 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4526
4527 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4528 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4529 }
4530
4531 /*
4532 * Guest LDTR.
4533 */
4534 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4535 {
4536 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4537 uint32_t u32Access = 0;
4538 if (!pMixedCtx->ldtr.Attr.u)
4539 u32Access = X86DESCATTR_UNUSABLE;
4540 else
4541 u32Access = pMixedCtx->ldtr.Attr.u;
4542
4543 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4544 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4545 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4546 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4547
4548 /* Validate. */
4549 if (!(u32Access & X86DESCATTR_UNUSABLE))
4550 {
4551 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4552 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4553 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4554 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4555 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4556 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4557 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4558 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4559 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4560 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4561 }
4562
4563 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4564 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4565 }
4566
4567 /*
4568 * Guest IDTR.
4569 */
4570 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4571 {
4572 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4573 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4574
4575 /* Validate. */
4576 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4577
4578 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4579 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4580 }
4581
4582 return VINF_SUCCESS;
4583}
4584
4585
4586/**
4587 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4588 * areas.
4589 *
4590 * These MSRs will automatically be loaded to the host CPU on every successful
4591 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4592 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4593 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4594 *
4595 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4596 *
4597 * @returns VBox status code.
4598 * @param pVCpu The cross context virtual CPU structure.
4599 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4600 * out-of-sync. Make sure to update the required fields
4601 * before using them.
4602 *
4603 * @remarks No-long-jump zone!!!
4604 */
4605static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4606{
4607 AssertPtr(pVCpu);
4608 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4609
4610 /*
4611 * MSRs that we use the auto-load/store MSR area in the VMCS.
4612 */
4613 PVM pVM = pVCpu->CTX_SUFF(pVM);
4614 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4615 {
4616 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4617#if HC_ARCH_BITS == 32
4618 if (pVM->hm.s.fAllow64BitGuests)
4619 {
4620 int rc = VINF_SUCCESS;
4621 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4622 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4623 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4624 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4625 AssertRCReturn(rc, rc);
4626# ifdef LOG_ENABLED
4627 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4628 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4629 {
4630 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4631 pMsr->u64Value));
4632 }
4633# endif
4634 }
4635#endif
4636 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4637 }
4638
4639 /*
4640 * Guest Sysenter MSRs.
4641 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4642 * VM-exits on WRMSRs for these MSRs.
4643 */
4644 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4645 {
4646 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4647 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4648 }
4649
4650 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4651 {
4652 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4653 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4654 }
4655
4656 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4657 {
4658 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4659 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4660 }
4661
4662 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4663 {
4664 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4665 {
4666 /*
4667 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4668 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4669 */
4670 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4671 {
4672 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4673 AssertRCReturn(rc,rc);
4674 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4675 }
4676 else
4677 {
4678 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4679 NULL /* pfAddedAndUpdated */);
4680 AssertRCReturn(rc, rc);
4681
4682 /* We need to intercept reads too, see @bugref{7386#c16}. */
4683 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4684 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4685 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4686 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4687 }
4688 }
4689 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4690 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4691 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4692 }
4693
4694 return VINF_SUCCESS;
4695}
4696
4697
4698/**
4699 * Loads the guest activity state into the guest-state area in the VMCS.
4700 *
4701 * @returns VBox status code.
4702 * @param pVCpu The cross context virtual CPU structure.
4703 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4704 * out-of-sync. Make sure to update the required fields
4705 * before using them.
4706 *
4707 * @remarks No-long-jump zone!!!
4708 */
4709static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4710{
4711 NOREF(pMixedCtx);
4712 /** @todo See if we can make use of other states, e.g.
4713 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4714 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4715 {
4716 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4717 AssertRCReturn(rc, rc);
4718
4719 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4720 }
4721 return VINF_SUCCESS;
4722}
4723
4724
4725/**
4726 * Sets up the appropriate function to run guest code.
4727 *
4728 * @returns VBox status code.
4729 * @param pVCpu The cross context virtual CPU structure.
4730 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4731 * out-of-sync. Make sure to update the required fields
4732 * before using them.
4733 *
4734 * @remarks No-long-jump zone!!!
4735 */
4736static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4737{
4738 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4739 {
4740#ifndef VBOX_ENABLE_64_BITS_GUESTS
4741 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4742#endif
4743 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4744#if HC_ARCH_BITS == 32
4745 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4746 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4747 {
4748 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4749 {
4750 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4751 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4752 | HM_CHANGED_VMX_ENTRY_CTLS
4753 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4754 }
4755 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4756 }
4757#else
4758 /* 64-bit host. */
4759 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4760#endif
4761 }
4762 else
4763 {
4764 /* Guest is not in long mode, use the 32-bit handler. */
4765#if HC_ARCH_BITS == 32
4766 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4767 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4768 {
4769 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4770 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4771 | HM_CHANGED_VMX_ENTRY_CTLS
4772 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4773 }
4774#endif
4775 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4776 }
4777 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4778 return VINF_SUCCESS;
4779}
4780
4781
4782/**
4783 * Wrapper for running the guest code in VT-x.
4784 *
4785 * @returns VBox strict status code.
4786 * @param pVM The cross context VM structure.
4787 * @param pVCpu The cross context virtual CPU structure.
4788 * @param pCtx Pointer to the guest-CPU context.
4789 *
4790 * @remarks No-long-jump zone!!!
4791 */
4792DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4793{
4794 /*
4795 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4796 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4797 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4798 */
4799 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4800 /** @todo Add stats for resume vs launch. */
4801#ifdef VBOX_WITH_KERNEL_USING_XMM
4802 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4803#else
4804 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4805#endif
4806}
4807
4808
4809/**
4810 * Reports world-switch error and dumps some useful debug info.
4811 *
4812 * @param pVM The cross context VM structure.
4813 * @param pVCpu The cross context virtual CPU structure.
4814 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4815 * @param pCtx Pointer to the guest-CPU context.
4816 * @param pVmxTransient Pointer to the VMX transient structure (only
4817 * exitReason updated).
4818 */
4819static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4820{
4821 Assert(pVM);
4822 Assert(pVCpu);
4823 Assert(pCtx);
4824 Assert(pVmxTransient);
4825 HMVMX_ASSERT_PREEMPT_SAFE();
4826
4827 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4828 switch (rcVMRun)
4829 {
4830 case VERR_VMX_INVALID_VMXON_PTR:
4831 AssertFailed();
4832 break;
4833 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4834 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4835 {
4836 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4837 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4838 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4839 AssertRC(rc);
4840
4841 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4842 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4843 Cannot do it here as we may have been long preempted. */
4844
4845#ifdef VBOX_STRICT
4846 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4847 pVmxTransient->uExitReason));
4848 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4849 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4850 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4851 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4852 else
4853 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4854 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4855 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4856
4857 /* VMX control bits. */
4858 uint32_t u32Val;
4859 uint64_t u64Val;
4860 RTHCUINTREG uHCReg;
4861 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4862 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4863 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4864 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4865 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4866 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4867 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4868 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4869 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4870 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4871 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4872 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4873 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4874 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4875 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4876 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4877 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4878 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4879 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4880 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4881 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4882 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4883 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4884 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4885 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4886 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4887 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4888 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4889 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4890 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4891 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4892 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4893 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4894 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4895 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4896 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4897 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4898 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4899 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4900 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4901 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4902 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4903
4904 /* Guest bits. */
4905 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4906 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4907 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4908 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4909 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4910 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4911 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4912 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4913
4914 /* Host bits. */
4915 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4916 Log4(("Host CR0 %#RHr\n", uHCReg));
4917 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4918 Log4(("Host CR3 %#RHr\n", uHCReg));
4919 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4920 Log4(("Host CR4 %#RHr\n", uHCReg));
4921
4922 RTGDTR HostGdtr;
4923 PCX86DESCHC pDesc;
4924 ASMGetGDTR(&HostGdtr);
4925 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4926 Log4(("Host CS %#08x\n", u32Val));
4927 if (u32Val < HostGdtr.cbGdt)
4928 {
4929 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4930 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4931 }
4932
4933 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4934 Log4(("Host DS %#08x\n", u32Val));
4935 if (u32Val < HostGdtr.cbGdt)
4936 {
4937 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4938 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4939 }
4940
4941 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4942 Log4(("Host ES %#08x\n", u32Val));
4943 if (u32Val < HostGdtr.cbGdt)
4944 {
4945 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4946 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4947 }
4948
4949 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4950 Log4(("Host FS %#08x\n", u32Val));
4951 if (u32Val < HostGdtr.cbGdt)
4952 {
4953 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4954 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4955 }
4956
4957 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4958 Log4(("Host GS %#08x\n", u32Val));
4959 if (u32Val < HostGdtr.cbGdt)
4960 {
4961 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4962 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4963 }
4964
4965 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4966 Log4(("Host SS %#08x\n", u32Val));
4967 if (u32Val < HostGdtr.cbGdt)
4968 {
4969 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4970 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4971 }
4972
4973 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4974 Log4(("Host TR %#08x\n", u32Val));
4975 if (u32Val < HostGdtr.cbGdt)
4976 {
4977 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4978 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4979 }
4980
4981 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4982 Log4(("Host TR Base %#RHv\n", uHCReg));
4983 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4984 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4985 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4986 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4987 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4988 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4989 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4990 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4991 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4992 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4993 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4994 Log4(("Host RSP %#RHv\n", uHCReg));
4995 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4996 Log4(("Host RIP %#RHv\n", uHCReg));
4997# if HC_ARCH_BITS == 64
4998 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4999 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5000 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5001 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5002 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5003 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5004# endif
5005#endif /* VBOX_STRICT */
5006 break;
5007 }
5008
5009 default:
5010 /* Impossible */
5011 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5012 break;
5013 }
5014 NOREF(pVM); NOREF(pCtx);
5015}
5016
5017
5018#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5019#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5020# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5021#endif
5022#ifdef VBOX_STRICT
5023static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5024{
5025 switch (idxField)
5026 {
5027 case VMX_VMCS_GUEST_RIP:
5028 case VMX_VMCS_GUEST_RSP:
5029 case VMX_VMCS_GUEST_SYSENTER_EIP:
5030 case VMX_VMCS_GUEST_SYSENTER_ESP:
5031 case VMX_VMCS_GUEST_GDTR_BASE:
5032 case VMX_VMCS_GUEST_IDTR_BASE:
5033 case VMX_VMCS_GUEST_CS_BASE:
5034 case VMX_VMCS_GUEST_DS_BASE:
5035 case VMX_VMCS_GUEST_ES_BASE:
5036 case VMX_VMCS_GUEST_FS_BASE:
5037 case VMX_VMCS_GUEST_GS_BASE:
5038 case VMX_VMCS_GUEST_SS_BASE:
5039 case VMX_VMCS_GUEST_LDTR_BASE:
5040 case VMX_VMCS_GUEST_TR_BASE:
5041 case VMX_VMCS_GUEST_CR3:
5042 return true;
5043 }
5044 return false;
5045}
5046
5047static bool hmR0VmxIsValidReadField(uint32_t idxField)
5048{
5049 switch (idxField)
5050 {
5051 /* Read-only fields. */
5052 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5053 return true;
5054 }
5055 /* Remaining readable fields should also be writable. */
5056 return hmR0VmxIsValidWriteField(idxField);
5057}
5058#endif /* VBOX_STRICT */
5059
5060
5061/**
5062 * Executes the specified handler in 64-bit mode.
5063 *
5064 * @returns VBox status code.
5065 * @param pVM The cross context VM structure.
5066 * @param pVCpu The cross context virtual CPU structure.
5067 * @param pCtx Pointer to the guest CPU context.
5068 * @param enmOp The operation to perform.
5069 * @param cParams Number of parameters.
5070 * @param paParam Array of 32-bit parameters.
5071 */
5072VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5073 uint32_t cParams, uint32_t *paParam)
5074{
5075 NOREF(pCtx);
5076
5077 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5078 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5079 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5080 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5081
5082#ifdef VBOX_STRICT
5083 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5084 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5085
5086 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5087 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5088#endif
5089
5090 /* Disable interrupts. */
5091 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5092
5093#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5094 RTCPUID idHostCpu = RTMpCpuId();
5095 CPUMR0SetLApic(pVCpu, idHostCpu);
5096#endif
5097
5098 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5099 RTHCPHYS HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5100
5101 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5102 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5103
5104 /* Leave VMX Root Mode. */
5105 VMXDisable();
5106
5107 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5108
5109 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5110 CPUMSetHyperEIP(pVCpu, enmOp);
5111 for (int i = (int)cParams - 1; i >= 0; i--)
5112 CPUMPushHyper(pVCpu, paParam[i]);
5113
5114 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5115
5116 /* Call the switcher. */
5117 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5118 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5119
5120 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5121 /* Make sure the VMX instructions don't cause #UD faults. */
5122 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5123
5124 /* Re-enter VMX Root Mode */
5125 int rc2 = VMXEnable(HCPhysCpuPage);
5126 if (RT_FAILURE(rc2))
5127 {
5128 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5129 ASMSetFlags(fOldEFlags);
5130 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5131 return rc2;
5132 }
5133
5134 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5135 AssertRC(rc2);
5136 Assert(!(ASMGetFlags() & X86_EFL_IF));
5137 ASMSetFlags(fOldEFlags);
5138 return rc;
5139}
5140
5141
5142/**
5143 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5144 * supporting 64-bit guests.
5145 *
5146 * @returns VBox status code.
5147 * @param fResume Whether to VMLAUNCH or VMRESUME.
5148 * @param pCtx Pointer to the guest-CPU context.
5149 * @param pCache Pointer to the VMCS cache.
5150 * @param pVM The cross context VM structure.
5151 * @param pVCpu The cross context virtual CPU structure.
5152 */
5153DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5154{
5155 NOREF(fResume);
5156
5157 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5158 RTHCPHYS HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5159
5160#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5161 pCache->uPos = 1;
5162 pCache->interPD = PGMGetInterPaeCR3(pVM);
5163 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5164#endif
5165
5166#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5167 pCache->TestIn.HCPhysCpuPage = 0;
5168 pCache->TestIn.HCPhysVmcs = 0;
5169 pCache->TestIn.pCache = 0;
5170 pCache->TestOut.HCPhysVmcs = 0;
5171 pCache->TestOut.pCache = 0;
5172 pCache->TestOut.pCtx = 0;
5173 pCache->TestOut.eflags = 0;
5174#else
5175 NOREF(pCache);
5176#endif
5177
5178 uint32_t aParam[10];
5179 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5180 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5181 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5182 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5183 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5184 aParam[5] = 0;
5185 aParam[6] = VM_RC_ADDR(pVM, pVM);
5186 aParam[7] = 0;
5187 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5188 aParam[9] = 0;
5189
5190#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5191 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5192 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5193#endif
5194 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5195
5196#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5197 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5198 Assert(pCtx->dr[4] == 10);
5199 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5200#endif
5201
5202#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5203 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5204 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5205 pVCpu->hm.s.vmx.HCPhysVmcs));
5206 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5207 pCache->TestOut.HCPhysVmcs));
5208 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5209 pCache->TestOut.pCache));
5210 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5211 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5212 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5213 pCache->TestOut.pCtx));
5214 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5215#endif
5216 return rc;
5217}
5218
5219
5220/**
5221 * Initialize the VMCS-Read cache.
5222 *
5223 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5224 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5225 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5226 * (those that have a 32-bit FULL & HIGH part).
5227 *
5228 * @returns VBox status code.
5229 * @param pVM The cross context VM structure.
5230 * @param pVCpu The cross context virtual CPU structure.
5231 */
5232static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5233{
5234#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5235{ \
5236 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5237 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5238 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5239 ++cReadFields; \
5240}
5241
5242 AssertPtr(pVM);
5243 AssertPtr(pVCpu);
5244 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5245 uint32_t cReadFields = 0;
5246
5247 /*
5248 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5249 * and serve to indicate exceptions to the rules.
5250 */
5251
5252 /* Guest-natural selector base fields. */
5253#if 0
5254 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5255 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5256 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5257#endif
5258 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5259 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5260 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5261 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5262 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5263 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5264 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5265 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5266 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5267 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5268 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5269 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5270#if 0
5271 /* Unused natural width guest-state fields. */
5272 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5273 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5274#endif
5275 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5276 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5277
5278 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5279#if 0
5280 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5281 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5282 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5283 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5284 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5285 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5286 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5287 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5288 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5289#endif
5290
5291 /* Natural width guest-state fields. */
5292 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5293#if 0
5294 /* Currently unused field. */
5295 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5296#endif
5297
5298 if (pVM->hm.s.fNestedPaging)
5299 {
5300 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5301 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5302 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5303 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5304 }
5305 else
5306 {
5307 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5308 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5309 }
5310
5311#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5312 return VINF_SUCCESS;
5313}
5314
5315
5316/**
5317 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5318 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5319 * darwin, running 64-bit guests).
5320 *
5321 * @returns VBox status code.
5322 * @param pVCpu The cross context virtual CPU structure.
5323 * @param idxField The VMCS field encoding.
5324 * @param u64Val 16, 32 or 64-bit value.
5325 */
5326VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5327{
5328 int rc;
5329 switch (idxField)
5330 {
5331 /*
5332 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5333 */
5334 /* 64-bit Control fields. */
5335 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5336 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5337 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5338 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5339 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5340 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5341 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5342 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5343 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5344 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5345 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5346 case VMX_VMCS64_CTRL_EPTP_FULL:
5347 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5348 /* 64-bit Guest-state fields. */
5349 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5350 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5351 case VMX_VMCS64_GUEST_PAT_FULL:
5352 case VMX_VMCS64_GUEST_EFER_FULL:
5353 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5354 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5355 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5356 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5357 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5358 /* 64-bit Host-state fields. */
5359 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5360 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5361 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5362 {
5363 rc = VMXWriteVmcs32(idxField, u64Val);
5364 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5365 break;
5366 }
5367
5368 /*
5369 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5370 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5371 */
5372 /* Natural-width Guest-state fields. */
5373 case VMX_VMCS_GUEST_CR3:
5374 case VMX_VMCS_GUEST_ES_BASE:
5375 case VMX_VMCS_GUEST_CS_BASE:
5376 case VMX_VMCS_GUEST_SS_BASE:
5377 case VMX_VMCS_GUEST_DS_BASE:
5378 case VMX_VMCS_GUEST_FS_BASE:
5379 case VMX_VMCS_GUEST_GS_BASE:
5380 case VMX_VMCS_GUEST_LDTR_BASE:
5381 case VMX_VMCS_GUEST_TR_BASE:
5382 case VMX_VMCS_GUEST_GDTR_BASE:
5383 case VMX_VMCS_GUEST_IDTR_BASE:
5384 case VMX_VMCS_GUEST_RSP:
5385 case VMX_VMCS_GUEST_RIP:
5386 case VMX_VMCS_GUEST_SYSENTER_ESP:
5387 case VMX_VMCS_GUEST_SYSENTER_EIP:
5388 {
5389 if (!(u64Val >> 32))
5390 {
5391 /* If this field is 64-bit, VT-x will zero out the top bits. */
5392 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5393 }
5394 else
5395 {
5396 /* Assert that only the 32->64 switcher case should ever come here. */
5397 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5398 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5399 }
5400 break;
5401 }
5402
5403 default:
5404 {
5405 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5406 rc = VERR_INVALID_PARAMETER;
5407 break;
5408 }
5409 }
5410 AssertRCReturn(rc, rc);
5411 return rc;
5412}
5413
5414
5415/**
5416 * Queue up a VMWRITE by using the VMCS write cache.
5417 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5418 *
5419 * @param pVCpu The cross context virtual CPU structure.
5420 * @param idxField The VMCS field encoding.
5421 * @param u64Val 16, 32 or 64-bit value.
5422 */
5423VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5424{
5425 AssertPtr(pVCpu);
5426 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5427
5428 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5429 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5430
5431 /* Make sure there are no duplicates. */
5432 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5433 {
5434 if (pCache->Write.aField[i] == idxField)
5435 {
5436 pCache->Write.aFieldVal[i] = u64Val;
5437 return VINF_SUCCESS;
5438 }
5439 }
5440
5441 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5442 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5443 pCache->Write.cValidEntries++;
5444 return VINF_SUCCESS;
5445}
5446#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5447
5448
5449/**
5450 * Sets up the usage of TSC-offsetting and updates the VMCS.
5451 *
5452 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5453 * VMX preemption timer.
5454 *
5455 * @returns VBox status code.
5456 * @param pVM The cross context VM structure.
5457 * @param pVCpu The cross context virtual CPU structure.
5458 *
5459 * @remarks No-long-jump zone!!!
5460 */
5461static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5462{
5463 int rc;
5464 bool fOffsettedTsc;
5465 bool fParavirtTsc;
5466 if (pVM->hm.s.vmx.fUsePreemptTimer)
5467 {
5468 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5469 &fOffsettedTsc, &fParavirtTsc);
5470
5471 /* Make sure the returned values have sane upper and lower boundaries. */
5472 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5473 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5474 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5475 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5476
5477 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5478 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5479 }
5480 else
5481 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5482
5483 /** @todo later optimize this to be done elsewhere and not before every
5484 * VM-entry. */
5485 if (fParavirtTsc)
5486 {
5487 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5488 information before every VM-entry, hence disable it for performance sake. */
5489#if 0
5490 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5491 AssertRC(rc);
5492#endif
5493 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5494 }
5495
5496 if (fOffsettedTsc)
5497 {
5498 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5499 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5500
5501 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5502 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5503 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5504 }
5505 else
5506 {
5507 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5508 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5509 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5510 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5511 }
5512}
5513
5514
5515/**
5516 * Determines if an exception is a contributory exception.
5517 *
5518 * Contributory exceptions are ones which can cause double-faults unless the
5519 * original exception was a benign exception. Page-fault is intentionally not
5520 * included here as it's a conditional contributory exception.
5521 *
5522 * @returns true if the exception is contributory, false otherwise.
5523 * @param uVector The exception vector.
5524 */
5525DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5526{
5527 switch (uVector)
5528 {
5529 case X86_XCPT_GP:
5530 case X86_XCPT_SS:
5531 case X86_XCPT_NP:
5532 case X86_XCPT_TS:
5533 case X86_XCPT_DE:
5534 return true;
5535 default:
5536 break;
5537 }
5538 return false;
5539}
5540
5541
5542/**
5543 * Sets an event as a pending event to be injected into the guest.
5544 *
5545 * @param pVCpu The cross context virtual CPU structure.
5546 * @param u32IntInfo The VM-entry interruption-information field.
5547 * @param cbInstr The VM-entry instruction length in bytes (for software
5548 * interrupts, exceptions and privileged software
5549 * exceptions).
5550 * @param u32ErrCode The VM-entry exception error code.
5551 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5552 * page-fault.
5553 *
5554 * @remarks Statistics counter assumes this is a guest event being injected or
5555 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5556 * always incremented.
5557 */
5558DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5559 RTGCUINTPTR GCPtrFaultAddress)
5560{
5561 Assert(!pVCpu->hm.s.Event.fPending);
5562 pVCpu->hm.s.Event.fPending = true;
5563 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5564 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5565 pVCpu->hm.s.Event.cbInstr = cbInstr;
5566 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5567
5568 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5569}
5570
5571
5572/**
5573 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5574 *
5575 * @param pVCpu The cross context virtual CPU structure.
5576 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5577 * out-of-sync. Make sure to update the required fields
5578 * before using them.
5579 */
5580DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5581{
5582 NOREF(pMixedCtx);
5583 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5584 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5585 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5586 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5587}
5588
5589
5590/**
5591 * Handle a condition that occurred while delivering an event through the guest
5592 * IDT.
5593 *
5594 * @returns VBox status code (informational error codes included).
5595 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5596 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought to
5597 * continue execution of the guest which will delivery the \#DF.
5598 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5599 *
5600 * @param pVCpu The cross context virtual CPU structure.
5601 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5602 * out-of-sync. Make sure to update the required fields
5603 * before using them.
5604 * @param pVmxTransient Pointer to the VMX transient structure.
5605 *
5606 * @remarks No-long-jump zone!!!
5607 */
5608static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5609{
5610 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5611
5612 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5613 AssertRCReturn(rc, rc);
5614 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5615 AssertRCReturn(rc, rc);
5616
5617 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5618 {
5619 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5620 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5621
5622 typedef enum
5623 {
5624 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5625 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5626 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5627 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5628 } VMXREFLECTXCPT;
5629
5630 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5631 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5632 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5633 {
5634 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5635 {
5636 enmReflect = VMXREFLECTXCPT_XCPT;
5637#ifdef VBOX_STRICT
5638 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5639 && uExitVector == X86_XCPT_PF)
5640 {
5641 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5642 }
5643#endif
5644 if ( uExitVector == X86_XCPT_PF
5645 && uIdtVector == X86_XCPT_PF)
5646 {
5647 pVmxTransient->fVectoringDoublePF = true;
5648 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5649 }
5650 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5651 && hmR0VmxIsContributoryXcpt(uExitVector)
5652 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5653 || uIdtVector == X86_XCPT_PF))
5654 {
5655 enmReflect = VMXREFLECTXCPT_DF;
5656 }
5657 else if (uIdtVector == X86_XCPT_DF)
5658 enmReflect = VMXREFLECTXCPT_TF;
5659 }
5660 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5661 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5662 {
5663 /*
5664 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5665 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5666 */
5667 enmReflect = VMXREFLECTXCPT_XCPT;
5668
5669 if (uExitVector == X86_XCPT_PF)
5670 {
5671 pVmxTransient->fVectoringPF = true;
5672 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5673 }
5674 }
5675 }
5676 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5677 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5678 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5679 {
5680 /*
5681 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5682 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5683 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5684 */
5685 enmReflect = VMXREFLECTXCPT_XCPT;
5686 }
5687
5688 /*
5689 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5690 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5691 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5692 *
5693 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5694 */
5695 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5696 && enmReflect == VMXREFLECTXCPT_XCPT
5697 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5698 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5699 {
5700 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5701 }
5702
5703 switch (enmReflect)
5704 {
5705 case VMXREFLECTXCPT_XCPT:
5706 {
5707 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5708 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5709 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5710
5711 uint32_t u32ErrCode = 0;
5712 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5713 {
5714 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5715 AssertRCReturn(rc, rc);
5716 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5717 }
5718
5719 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5720 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5721 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5722 rc = VINF_SUCCESS;
5723 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5724 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5725
5726 break;
5727 }
5728
5729 case VMXREFLECTXCPT_DF:
5730 {
5731 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5732 rc = VINF_HM_DOUBLE_FAULT;
5733 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5734 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5735
5736 break;
5737 }
5738
5739 case VMXREFLECTXCPT_TF:
5740 {
5741 rc = VINF_EM_RESET;
5742 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5743 uExitVector));
5744 break;
5745 }
5746
5747 default:
5748 Assert(rc == VINF_SUCCESS);
5749 break;
5750 }
5751 }
5752 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5753 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5754 && uExitVector != X86_XCPT_DF
5755 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5756 {
5757 /*
5758 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5759 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5760 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5761 */
5762 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5763 {
5764 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5765 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5766 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5767 }
5768 }
5769
5770 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5771 return rc;
5772}
5773
5774
5775/**
5776 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5777 *
5778 * @returns VBox status code.
5779 * @param pVCpu The cross context virtual CPU structure.
5780 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5781 * out-of-sync. Make sure to update the required fields
5782 * before using them.
5783 *
5784 * @remarks No-long-jump zone!!!
5785 */
5786static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5787{
5788 NOREF(pMixedCtx);
5789
5790 /*
5791 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5792 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5793 */
5794 VMMRZCallRing3Disable(pVCpu);
5795 HM_DISABLE_PREEMPT();
5796
5797 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5798 {
5799 uint32_t uVal = 0;
5800 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5801 AssertRCReturn(rc, rc);
5802
5803 uint32_t uShadow = 0;
5804 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5805 AssertRCReturn(rc, rc);
5806
5807 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5808 CPUMSetGuestCR0(pVCpu, uVal);
5809 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5810 }
5811
5812 HM_RESTORE_PREEMPT();
5813 VMMRZCallRing3Enable(pVCpu);
5814 return VINF_SUCCESS;
5815}
5816
5817
5818/**
5819 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5820 *
5821 * @returns VBox status code.
5822 * @param pVCpu The cross context virtual CPU structure.
5823 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5824 * out-of-sync. Make sure to update the required fields
5825 * before using them.
5826 *
5827 * @remarks No-long-jump zone!!!
5828 */
5829static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5830{
5831 NOREF(pMixedCtx);
5832
5833 int rc = VINF_SUCCESS;
5834 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5835 {
5836 uint32_t uVal = 0;
5837 uint32_t uShadow = 0;
5838 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5839 AssertRCReturn(rc, rc);
5840 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5841 AssertRCReturn(rc, rc);
5842
5843 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5844 CPUMSetGuestCR4(pVCpu, uVal);
5845 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5846 }
5847 return rc;
5848}
5849
5850
5851/**
5852 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5853 *
5854 * @returns VBox status code.
5855 * @param pVCpu The cross context virtual CPU structure.
5856 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5857 * out-of-sync. Make sure to update the required fields
5858 * before using them.
5859 *
5860 * @remarks No-long-jump zone!!!
5861 */
5862static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5863{
5864 int rc = VINF_SUCCESS;
5865 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5866 {
5867 uint64_t u64Val = 0;
5868 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5869 AssertRCReturn(rc, rc);
5870
5871 pMixedCtx->rip = u64Val;
5872 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5873 }
5874 return rc;
5875}
5876
5877
5878/**
5879 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5880 *
5881 * @returns VBox status code.
5882 * @param pVCpu The cross context virtual CPU structure.
5883 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5884 * out-of-sync. Make sure to update the required fields
5885 * before using them.
5886 *
5887 * @remarks No-long-jump zone!!!
5888 */
5889static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5890{
5891 int rc = VINF_SUCCESS;
5892 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5893 {
5894 uint64_t u64Val = 0;
5895 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5896 AssertRCReturn(rc, rc);
5897
5898 pMixedCtx->rsp = u64Val;
5899 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5900 }
5901 return rc;
5902}
5903
5904
5905/**
5906 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5907 *
5908 * @returns VBox status code.
5909 * @param pVCpu The cross context virtual CPU structure.
5910 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5911 * out-of-sync. Make sure to update the required fields
5912 * before using them.
5913 *
5914 * @remarks No-long-jump zone!!!
5915 */
5916static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5917{
5918 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5919 {
5920 uint32_t uVal = 0;
5921 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5922 AssertRCReturn(rc, rc);
5923
5924 pMixedCtx->eflags.u32 = uVal;
5925 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5926 {
5927 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5928 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5929
5930 pMixedCtx->eflags.Bits.u1VM = 0;
5931 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5932 }
5933
5934 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5935 }
5936 return VINF_SUCCESS;
5937}
5938
5939
5940/**
5941 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5942 * guest-CPU context.
5943 */
5944DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5945{
5946 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5947 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5948 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5949 return rc;
5950}
5951
5952
5953/**
5954 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5955 * from the guest-state area in the VMCS.
5956 *
5957 * @param pVCpu The cross context virtual CPU structure.
5958 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5959 * out-of-sync. Make sure to update the required fields
5960 * before using them.
5961 *
5962 * @remarks No-long-jump zone!!!
5963 */
5964static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5965{
5966 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
5967 {
5968 uint32_t uIntrState = 0;
5969 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5970 AssertRC(rc);
5971
5972 if (!uIntrState)
5973 {
5974 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5975 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5976
5977 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5978 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5979 }
5980 else
5981 {
5982 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
5983 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
5984 {
5985 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5986 AssertRC(rc);
5987 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5988 AssertRC(rc);
5989
5990 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5991 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5992 }
5993 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5994 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5995
5996 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
5997 {
5998 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5999 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6000 }
6001 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6002 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6003 }
6004
6005 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6006 }
6007}
6008
6009
6010/**
6011 * Saves the guest's activity state.
6012 *
6013 * @returns VBox status code.
6014 * @param pVCpu The cross context virtual CPU structure.
6015 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6016 * out-of-sync. Make sure to update the required fields
6017 * before using them.
6018 *
6019 * @remarks No-long-jump zone!!!
6020 */
6021static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6022{
6023 NOREF(pMixedCtx);
6024 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6025 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6026 return VINF_SUCCESS;
6027}
6028
6029
6030/**
6031 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6032 * the current VMCS into the guest-CPU context.
6033 *
6034 * @returns VBox status code.
6035 * @param pVCpu The cross context virtual CPU structure.
6036 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6037 * out-of-sync. Make sure to update the required fields
6038 * before using them.
6039 *
6040 * @remarks No-long-jump zone!!!
6041 */
6042static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6043{
6044 int rc = VINF_SUCCESS;
6045 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6046 {
6047 uint32_t u32Val = 0;
6048 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6049 pMixedCtx->SysEnter.cs = u32Val;
6050 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6051 }
6052
6053 uint64_t u64Val = 0;
6054 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6055 {
6056 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6057 pMixedCtx->SysEnter.eip = u64Val;
6058 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6059 }
6060 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6061 {
6062 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6063 pMixedCtx->SysEnter.esp = u64Val;
6064 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6065 }
6066 return rc;
6067}
6068
6069
6070/**
6071 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6072 * the CPU back into the guest-CPU context.
6073 *
6074 * @returns VBox status code.
6075 * @param pVCpu The cross context virtual CPU structure.
6076 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6077 * out-of-sync. Make sure to update the required fields
6078 * before using them.
6079 *
6080 * @remarks No-long-jump zone!!!
6081 */
6082static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6083{
6084#if HC_ARCH_BITS == 64
6085 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6086 {
6087 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6088 VMMRZCallRing3Disable(pVCpu);
6089 HM_DISABLE_PREEMPT();
6090
6091 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6092 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6093 {
6094 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6095 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6096 }
6097
6098 HM_RESTORE_PREEMPT();
6099 VMMRZCallRing3Enable(pVCpu);
6100 }
6101 else
6102 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6103#else
6104 NOREF(pMixedCtx);
6105 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6106#endif
6107
6108 return VINF_SUCCESS;
6109}
6110
6111
6112/**
6113 * Saves the auto load/store'd guest MSRs from the current VMCS into
6114 * the guest-CPU context.
6115 *
6116 * @returns VBox status code.
6117 * @param pVCpu The cross context virtual CPU structure.
6118 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6119 * out-of-sync. Make sure to update the required fields
6120 * before using them.
6121 *
6122 * @remarks No-long-jump zone!!!
6123 */
6124static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6125{
6126 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6127 return VINF_SUCCESS;
6128
6129 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6130 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6131 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6132 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6133 {
6134 switch (pMsr->u32Msr)
6135 {
6136 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6137 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6138 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6139 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6140 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6141 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6142 break;
6143
6144 default:
6145 {
6146 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6147 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6148 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6149 }
6150 }
6151 }
6152
6153 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6154 return VINF_SUCCESS;
6155}
6156
6157
6158/**
6159 * Saves the guest control registers from the current VMCS into the guest-CPU
6160 * context.
6161 *
6162 * @returns VBox status code.
6163 * @param pVCpu The cross context virtual CPU structure.
6164 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6165 * out-of-sync. Make sure to update the required fields
6166 * before using them.
6167 *
6168 * @remarks No-long-jump zone!!!
6169 */
6170static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6171{
6172 /* Guest CR0. Guest FPU. */
6173 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6174 AssertRCReturn(rc, rc);
6175
6176 /* Guest CR4. */
6177 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6178 AssertRCReturn(rc, rc);
6179
6180 /* Guest CR2 - updated always during the world-switch or in #PF. */
6181 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6182 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6183 {
6184 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6185 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6186
6187 PVM pVM = pVCpu->CTX_SUFF(pVM);
6188 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6189 || ( pVM->hm.s.fNestedPaging
6190 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6191 {
6192 uint64_t u64Val = 0;
6193 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6194 if (pMixedCtx->cr3 != u64Val)
6195 {
6196 CPUMSetGuestCR3(pVCpu, u64Val);
6197 if (VMMRZCallRing3IsEnabled(pVCpu))
6198 {
6199 PGMUpdateCR3(pVCpu, u64Val);
6200 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6201 }
6202 else
6203 {
6204 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6205 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6206 }
6207 }
6208
6209 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6210 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6211 {
6212 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6213 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6214 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6215 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6216
6217 if (VMMRZCallRing3IsEnabled(pVCpu))
6218 {
6219 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6220 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6221 }
6222 else
6223 {
6224 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6225 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6226 }
6227 }
6228 }
6229
6230 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6231 }
6232
6233 /*
6234 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6235 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6236 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6237 *
6238 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6239 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6240 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6241 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6242 *
6243 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6244 */
6245 if (VMMRZCallRing3IsEnabled(pVCpu))
6246 {
6247 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6248 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6249
6250 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6251 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6252
6253 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6254 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6255 }
6256
6257 return rc;
6258}
6259
6260
6261/**
6262 * Reads a guest segment register from the current VMCS into the guest-CPU
6263 * context.
6264 *
6265 * @returns VBox status code.
6266 * @param pVCpu The cross context virtual CPU structure.
6267 * @param idxSel Index of the selector in the VMCS.
6268 * @param idxLimit Index of the segment limit in the VMCS.
6269 * @param idxBase Index of the segment base in the VMCS.
6270 * @param idxAccess Index of the access rights of the segment in the VMCS.
6271 * @param pSelReg Pointer to the segment selector.
6272 *
6273 * @remarks No-long-jump zone!!!
6274 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6275 * macro as that takes care of whether to read from the VMCS cache or
6276 * not.
6277 */
6278DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6279 PCPUMSELREG pSelReg)
6280{
6281 NOREF(pVCpu);
6282
6283 uint32_t u32Val = 0;
6284 int rc = VMXReadVmcs32(idxSel, &u32Val);
6285 AssertRCReturn(rc, rc);
6286 pSelReg->Sel = (uint16_t)u32Val;
6287 pSelReg->ValidSel = (uint16_t)u32Val;
6288 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6289
6290 rc = VMXReadVmcs32(idxLimit, &u32Val);
6291 AssertRCReturn(rc, rc);
6292 pSelReg->u32Limit = u32Val;
6293
6294 uint64_t u64Val = 0;
6295 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6296 AssertRCReturn(rc, rc);
6297 pSelReg->u64Base = u64Val;
6298
6299 rc = VMXReadVmcs32(idxAccess, &u32Val);
6300 AssertRCReturn(rc, rc);
6301 pSelReg->Attr.u = u32Val;
6302
6303 /*
6304 * If VT-x marks the segment as unusable, most other bits remain undefined:
6305 * - For CS the L, D and G bits have meaning.
6306 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6307 * - For the remaining data segments no bits are defined.
6308 *
6309 * The present bit and the unusable bit has been observed to be set at the
6310 * same time (the selector was supposed to be invalid as we started executing
6311 * a V8086 interrupt in ring-0).
6312 *
6313 * What should be important for the rest of the VBox code, is that the P bit is
6314 * cleared. Some of the other VBox code recognizes the unusable bit, but
6315 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6316 * safe side here, we'll strip off P and other bits we don't care about. If
6317 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6318 *
6319 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6320 */
6321 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6322 {
6323 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6324
6325 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6326 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6327 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6328
6329 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6330#ifdef DEBUG_bird
6331 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6332 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6333 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6334#endif
6335 }
6336 return VINF_SUCCESS;
6337}
6338
6339
6340#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6341# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6342 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6343 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6344#else
6345# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6346 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6347 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6348#endif
6349
6350
6351/**
6352 * Saves the guest segment registers from the current VMCS into the guest-CPU
6353 * context.
6354 *
6355 * @returns VBox status code.
6356 * @param pVCpu The cross context virtual CPU structure.
6357 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6358 * out-of-sync. Make sure to update the required fields
6359 * before using them.
6360 *
6361 * @remarks No-long-jump zone!!!
6362 */
6363static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6364{
6365 /* Guest segment registers. */
6366 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6367 {
6368 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6369 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6370 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6371 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6372 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6373 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6374 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6375
6376 /* Restore segment attributes for real-on-v86 mode hack. */
6377 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6378 {
6379 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6380 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6381 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6382 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6383 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6384 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6385 }
6386 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6387 }
6388
6389 return VINF_SUCCESS;
6390}
6391
6392
6393/**
6394 * Saves the guest descriptor table registers and task register from the current
6395 * VMCS into the guest-CPU context.
6396 *
6397 * @returns VBox status code.
6398 * @param pVCpu The cross context virtual CPU structure.
6399 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6400 * out-of-sync. Make sure to update the required fields
6401 * before using them.
6402 *
6403 * @remarks No-long-jump zone!!!
6404 */
6405static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6406{
6407 int rc = VINF_SUCCESS;
6408
6409 /* Guest LDTR. */
6410 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6411 {
6412 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6413 AssertRCReturn(rc, rc);
6414 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6415 }
6416
6417 /* Guest GDTR. */
6418 uint64_t u64Val = 0;
6419 uint32_t u32Val = 0;
6420 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6421 {
6422 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6423 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6424 pMixedCtx->gdtr.pGdt = u64Val;
6425 pMixedCtx->gdtr.cbGdt = u32Val;
6426 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6427 }
6428
6429 /* Guest IDTR. */
6430 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6431 {
6432 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6433 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6434 pMixedCtx->idtr.pIdt = u64Val;
6435 pMixedCtx->idtr.cbIdt = u32Val;
6436 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6437 }
6438
6439 /* Guest TR. */
6440 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6441 {
6442 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6443 AssertRCReturn(rc, rc);
6444
6445 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6446 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6447 {
6448 rc = VMXLOCAL_READ_SEG(TR, tr);
6449 AssertRCReturn(rc, rc);
6450 }
6451 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6452 }
6453 return rc;
6454}
6455
6456#undef VMXLOCAL_READ_SEG
6457
6458
6459/**
6460 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6461 * context.
6462 *
6463 * @returns VBox status code.
6464 * @param pVCpu The cross context virtual CPU structure.
6465 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6466 * out-of-sync. Make sure to update the required fields
6467 * before using them.
6468 *
6469 * @remarks No-long-jump zone!!!
6470 */
6471static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6472{
6473 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6474 {
6475 if (!pVCpu->hm.s.fUsingHyperDR7)
6476 {
6477 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6478 uint32_t u32Val;
6479 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6480 pMixedCtx->dr[7] = u32Val;
6481 }
6482
6483 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6484 }
6485 return VINF_SUCCESS;
6486}
6487
6488
6489/**
6490 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6491 *
6492 * @returns VBox status code.
6493 * @param pVCpu The cross context virtual CPU structure.
6494 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6495 * out-of-sync. Make sure to update the required fields
6496 * before using them.
6497 *
6498 * @remarks No-long-jump zone!!!
6499 */
6500static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6501{
6502 NOREF(pMixedCtx);
6503
6504 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6505 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6506 return VINF_SUCCESS;
6507}
6508
6509
6510/**
6511 * Saves the entire guest state from the currently active VMCS into the
6512 * guest-CPU context.
6513 *
6514 * This essentially VMREADs all guest-data.
6515 *
6516 * @returns VBox status code.
6517 * @param pVCpu The cross context virtual CPU structure.
6518 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6519 * out-of-sync. Make sure to update the required fields
6520 * before using them.
6521 */
6522static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6523{
6524 Assert(pVCpu);
6525 Assert(pMixedCtx);
6526
6527 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6528 return VINF_SUCCESS;
6529
6530 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6531 again on the ring-3 callback path, there is no real need to. */
6532 if (VMMRZCallRing3IsEnabled(pVCpu))
6533 VMMR0LogFlushDisable(pVCpu);
6534 else
6535 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6536 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6537
6538 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6539 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6540
6541 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6542 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6543
6544 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6545 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6546
6547 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6548 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6549
6550 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6551 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6552
6553 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6554 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6555
6556 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6557 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6558
6559 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6560 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6561
6562 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6563 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6564
6565 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6566 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6567
6568 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6569 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6570
6571 if (VMMRZCallRing3IsEnabled(pVCpu))
6572 VMMR0LogFlushEnable(pVCpu);
6573
6574 return VINF_SUCCESS;
6575}
6576
6577
6578/**
6579 * Saves basic guest registers needed for IEM instruction execution.
6580 *
6581 * @returns VBox status code (OR-able).
6582 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6583 * @param pMixedCtx Pointer to the CPU context of the guest.
6584 * @param fMemory Whether the instruction being executed operates on
6585 * memory or not. Only CR0 is synced up if clear.
6586 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6587 */
6588static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6589{
6590 /*
6591 * We assume all general purpose registers other than RSP are available.
6592 *
6593 * RIP is a must, as it will be incremented or otherwise changed.
6594 *
6595 * RFLAGS are always required to figure the CPL.
6596 *
6597 * RSP isn't always required, however it's a GPR, so frequently required.
6598 *
6599 * SS and CS are the only segment register needed if IEM doesn't do memory
6600 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6601 *
6602 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6603 * be required for memory accesses.
6604 *
6605 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6606 */
6607 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6608 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6609 if (fNeedRsp)
6610 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6611 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6612 if (!fMemory)
6613 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6614 else
6615 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6616 return rc;
6617}
6618
6619
6620/**
6621 * Ensures that we've got a complete basic guest-context.
6622 *
6623 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6624 * is for the interpreter.
6625 *
6626 * @returns VBox status code.
6627 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6628 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6629 * needing to be synced in.
6630 * @thread EMT(pVCpu)
6631 */
6632VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6633{
6634 /* Note! Since this is only applicable to VT-x, the implementation is placed
6635 in the VT-x part of the sources instead of the generic stuff. */
6636 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6637 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6638 return VINF_SUCCESS;
6639}
6640
6641
6642/**
6643 * Check per-VM and per-VCPU force flag actions that require us to go back to
6644 * ring-3 for one reason or another.
6645 *
6646 * @returns VBox status code (information status code included).
6647 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6648 * ring-3.
6649 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6650 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6651 * interrupts)
6652 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6653 * all EMTs to be in ring-3.
6654 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6655 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6656 * to the EM loop.
6657 *
6658 * @param pVM The cross context VM structure.
6659 * @param pVCpu The cross context virtual CPU structure.
6660 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6661 * out-of-sync. Make sure to update the required fields
6662 * before using them.
6663 */
6664static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6665{
6666 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6667
6668 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6669 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6670 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6671 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6672 {
6673 /* We need the control registers now, make sure the guest-CPU context is updated. */
6674 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6675 AssertRCReturn(rc3, rc3);
6676
6677 /* Pending HM CR3 sync. */
6678 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6679 {
6680 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6681 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6682 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6683 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6684 }
6685
6686 /* Pending HM PAE PDPEs. */
6687 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6688 {
6689 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6690 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6691 }
6692
6693 /* Pending PGM C3 sync. */
6694 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6695 {
6696 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6697 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6698 if (rc2 != VINF_SUCCESS)
6699 {
6700 AssertRC(rc2);
6701 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6702 return rc2;
6703 }
6704 }
6705
6706 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6707 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6708 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6709 {
6710 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6711 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6712 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6713 return rc2;
6714 }
6715
6716 /* Pending VM request packets, such as hardware interrupts. */
6717 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6718 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6719 {
6720 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6721 return VINF_EM_PENDING_REQUEST;
6722 }
6723
6724 /* Pending PGM pool flushes. */
6725 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6726 {
6727 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6728 return VINF_PGM_POOL_FLUSH_PENDING;
6729 }
6730
6731 /* Pending DMA requests. */
6732 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6733 {
6734 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6735 return VINF_EM_RAW_TO_R3;
6736 }
6737 }
6738
6739 return VINF_SUCCESS;
6740}
6741
6742
6743/**
6744 * Converts any TRPM trap into a pending HM event. This is typically used when
6745 * entering from ring-3 (not longjmp returns).
6746 *
6747 * @param pVCpu The cross context virtual CPU structure.
6748 */
6749static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6750{
6751 Assert(TRPMHasTrap(pVCpu));
6752 Assert(!pVCpu->hm.s.Event.fPending);
6753
6754 uint8_t uVector;
6755 TRPMEVENT enmTrpmEvent;
6756 RTGCUINT uErrCode;
6757 RTGCUINTPTR GCPtrFaultAddress;
6758 uint8_t cbInstr;
6759
6760 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6761 AssertRC(rc);
6762
6763 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6764 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6765 if (enmTrpmEvent == TRPM_TRAP)
6766 {
6767 switch (uVector)
6768 {
6769 case X86_XCPT_NMI:
6770 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6771 break;
6772
6773 case X86_XCPT_BP:
6774 case X86_XCPT_OF:
6775 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6776 break;
6777
6778 case X86_XCPT_PF:
6779 case X86_XCPT_DF:
6780 case X86_XCPT_TS:
6781 case X86_XCPT_NP:
6782 case X86_XCPT_SS:
6783 case X86_XCPT_GP:
6784 case X86_XCPT_AC:
6785 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6786 /* no break! */
6787 default:
6788 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6789 break;
6790 }
6791 }
6792 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6793 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6794 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6795 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6796 else
6797 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6798
6799 rc = TRPMResetTrap(pVCpu);
6800 AssertRC(rc);
6801 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6802 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6803
6804 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6805 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6806}
6807
6808
6809/**
6810 * Converts the pending HM event into a TRPM trap.
6811 *
6812 * @param pVCpu The cross context virtual CPU structure.
6813 */
6814static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6815{
6816 Assert(pVCpu->hm.s.Event.fPending);
6817
6818 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6819 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6820 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6821 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6822
6823 /* If a trap was already pending, we did something wrong! */
6824 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6825
6826 TRPMEVENT enmTrapType;
6827 switch (uVectorType)
6828 {
6829 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6830 enmTrapType = TRPM_HARDWARE_INT;
6831 break;
6832
6833 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6834 enmTrapType = TRPM_SOFTWARE_INT;
6835 break;
6836
6837 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6838 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6839 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6840 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6841 enmTrapType = TRPM_TRAP;
6842 break;
6843
6844 default:
6845 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6846 enmTrapType = TRPM_32BIT_HACK;
6847 break;
6848 }
6849
6850 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6851
6852 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6853 AssertRC(rc);
6854
6855 if (fErrorCodeValid)
6856 TRPMSetErrorCode(pVCpu, uErrorCode);
6857
6858 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6859 && uVector == X86_XCPT_PF)
6860 {
6861 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6862 }
6863 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6864 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6865 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6866 {
6867 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6868 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6869 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6870 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6871 }
6872
6873 /* Clear any pending events from the VMCS. */
6874 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6875 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6876
6877 /* We're now done converting the pending event. */
6878 pVCpu->hm.s.Event.fPending = false;
6879}
6880
6881
6882/**
6883 * Does the necessary state syncing before returning to ring-3 for any reason
6884 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6885 *
6886 * @returns VBox status code.
6887 * @param pVM The cross context VM structure.
6888 * @param pVCpu The cross context virtual CPU structure.
6889 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6890 * be out-of-sync. Make sure to update the required
6891 * fields before using them.
6892 * @param fSaveGuestState Whether to save the guest state or not.
6893 *
6894 * @remarks No-long-jmp zone!!!
6895 */
6896static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6897{
6898 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6899 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6900
6901 RTCPUID idCpu = RTMpCpuId();
6902 Log4Func(("HostCpuId=%u\n", idCpu));
6903
6904 /*
6905 * !!! IMPORTANT !!!
6906 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6907 */
6908
6909 /* Save the guest state if necessary. */
6910 if ( fSaveGuestState
6911 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6912 {
6913 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6914 AssertRCReturn(rc, rc);
6915 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6916 }
6917
6918 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6919 if (CPUMIsGuestFPUStateActive(pVCpu))
6920 {
6921 /* We shouldn't reload CR0 without saving it first. */
6922 if (!fSaveGuestState)
6923 {
6924 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6925 AssertRCReturn(rc, rc);
6926 }
6927 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6928 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6929 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6930 }
6931
6932 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6933#ifdef VBOX_STRICT
6934 if (CPUMIsHyperDebugStateActive(pVCpu))
6935 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6936#endif
6937 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6938 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6939 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6940 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6941
6942#if HC_ARCH_BITS == 64
6943 /* Restore host-state bits that VT-x only restores partially. */
6944 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6945 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6946 {
6947 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6948 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6949 }
6950 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6951#endif
6952
6953#if HC_ARCH_BITS == 64
6954 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6955 if ( pVM->hm.s.fAllow64BitGuests
6956 && pVCpu->hm.s.vmx.fLazyMsrs)
6957 {
6958 /* We shouldn't reload the guest MSRs without saving it first. */
6959 if (!fSaveGuestState)
6960 {
6961 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6962 AssertRCReturn(rc, rc);
6963 }
6964 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6965 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6966 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6967 }
6968#endif
6969
6970 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6971 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6972
6973 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6974 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6975 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6976 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6977 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6978 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6979 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6980 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6981
6982 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6983
6984 /** @todo This partially defeats the purpose of having preemption hooks.
6985 * The problem is, deregistering the hooks should be moved to a place that
6986 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6987 * context.
6988 */
6989 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6990 {
6991 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6992 AssertRCReturn(rc, rc);
6993
6994 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6995 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6996 }
6997 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6998 NOREF(idCpu);
6999
7000 return VINF_SUCCESS;
7001}
7002
7003
7004/**
7005 * Leaves the VT-x session.
7006 *
7007 * @returns VBox status code.
7008 * @param pVM The cross context VM structure.
7009 * @param pVCpu The cross context virtual CPU structure.
7010 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7011 * out-of-sync. Make sure to update the required fields
7012 * before using them.
7013 *
7014 * @remarks No-long-jmp zone!!!
7015 */
7016DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7017{
7018 HM_DISABLE_PREEMPT();
7019 HMVMX_ASSERT_CPU_SAFE();
7020 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7021 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7022
7023 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7024 and done this from the VMXR0ThreadCtxCallback(). */
7025 if (!pVCpu->hm.s.fLeaveDone)
7026 {
7027 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7028 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7029 pVCpu->hm.s.fLeaveDone = true;
7030 }
7031 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7032
7033 /*
7034 * !!! IMPORTANT !!!
7035 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7036 */
7037
7038 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7039 /** @todo Deregistering here means we need to VMCLEAR always
7040 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7041 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7042 VMMR0ThreadCtxHookDisable(pVCpu);
7043
7044 /* Leave HM context. This takes care of local init (term). */
7045 int rc = HMR0LeaveCpu(pVCpu);
7046
7047 HM_RESTORE_PREEMPT();
7048 return rc;
7049}
7050
7051
7052/**
7053 * Does the necessary state syncing before doing a longjmp to ring-3.
7054 *
7055 * @returns VBox status code.
7056 * @param pVM The cross context VM structure.
7057 * @param pVCpu The cross context virtual CPU structure.
7058 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7059 * out-of-sync. Make sure to update the required fields
7060 * before using them.
7061 *
7062 * @remarks No-long-jmp zone!!!
7063 */
7064DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7065{
7066 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7067}
7068
7069
7070/**
7071 * Take necessary actions before going back to ring-3.
7072 *
7073 * An action requires us to go back to ring-3. This function does the necessary
7074 * steps before we can safely return to ring-3. This is not the same as longjmps
7075 * to ring-3, this is voluntary and prepares the guest so it may continue
7076 * executing outside HM (recompiler/IEM).
7077 *
7078 * @returns VBox status code.
7079 * @param pVM The cross context VM structure.
7080 * @param pVCpu The cross context virtual CPU structure.
7081 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7082 * out-of-sync. Make sure to update the required fields
7083 * before using them.
7084 * @param rcExit The reason for exiting to ring-3. Can be
7085 * VINF_VMM_UNKNOWN_RING3_CALL.
7086 */
7087static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7088{
7089 Assert(pVM);
7090 Assert(pVCpu);
7091 Assert(pMixedCtx);
7092 HMVMX_ASSERT_PREEMPT_SAFE();
7093
7094 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7095 {
7096 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7097 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7098 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7099 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7100 }
7101
7102 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7103 VMMRZCallRing3Disable(pVCpu);
7104 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7105
7106 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7107 if (pVCpu->hm.s.Event.fPending)
7108 {
7109 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7110 Assert(!pVCpu->hm.s.Event.fPending);
7111 }
7112
7113 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7114 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7115
7116 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7117 and if we're injecting an event we should have a TRPM trap pending. */
7118 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", rcExit));
7119 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", rcExit));
7120
7121 /* Save guest state and restore host state bits. */
7122 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7123 AssertRCReturn(rc, rc);
7124 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7125 /* Thread-context hooks are unregistered at this point!!! */
7126
7127 /* Sync recompiler state. */
7128 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7129 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7130 | CPUM_CHANGED_LDTR
7131 | CPUM_CHANGED_GDTR
7132 | CPUM_CHANGED_IDTR
7133 | CPUM_CHANGED_TR
7134 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7135 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7136 if ( pVM->hm.s.fNestedPaging
7137 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7138 {
7139 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7140 }
7141
7142 Assert(!pVCpu->hm.s.fClearTrapFlag);
7143
7144 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7145 if (rcExit != VINF_EM_RAW_INTERRUPT)
7146 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7147
7148 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7149
7150 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7151 VMMRZCallRing3RemoveNotification(pVCpu);
7152 VMMRZCallRing3Enable(pVCpu);
7153
7154 return rc;
7155}
7156
7157
7158/**
7159 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7160 * longjump to ring-3 and possibly get preempted.
7161 *
7162 * @returns VBox status code.
7163 * @param pVCpu The cross context virtual CPU structure.
7164 * @param enmOperation The operation causing the ring-3 longjump.
7165 * @param pvUser Opaque pointer to the guest-CPU context. The data
7166 * may be out-of-sync. Make sure to update the required
7167 * fields before using them.
7168 */
7169static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7170{
7171 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7172 {
7173 /*
7174 * !!! IMPORTANT !!!
7175 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7176 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7177 */
7178 VMMRZCallRing3RemoveNotification(pVCpu);
7179 VMMRZCallRing3Disable(pVCpu);
7180 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7181 RTThreadPreemptDisable(&PreemptState);
7182
7183 PVM pVM = pVCpu->CTX_SUFF(pVM);
7184 if (CPUMIsGuestFPUStateActive(pVCpu))
7185 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7186
7187 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7188
7189#if HC_ARCH_BITS == 64
7190 /* Restore host-state bits that VT-x only restores partially. */
7191 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7192 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7193 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7194 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7195
7196 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7197 if ( pVM->hm.s.fAllow64BitGuests
7198 && pVCpu->hm.s.vmx.fLazyMsrs)
7199 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7200#endif
7201 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7202 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7203 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7204 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7205 {
7206 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7207 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7208 }
7209
7210 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7211 VMMR0ThreadCtxHookDisable(pVCpu);
7212 HMR0LeaveCpu(pVCpu);
7213 RTThreadPreemptRestore(&PreemptState);
7214 return VINF_SUCCESS;
7215 }
7216
7217 Assert(pVCpu);
7218 Assert(pvUser);
7219 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7220 HMVMX_ASSERT_PREEMPT_SAFE();
7221
7222 VMMRZCallRing3Disable(pVCpu);
7223 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7224
7225 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7226 enmOperation));
7227
7228 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7229 AssertRCReturn(rc, rc);
7230
7231 VMMRZCallRing3Enable(pVCpu);
7232 return VINF_SUCCESS;
7233}
7234
7235
7236/**
7237 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7238 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7239 *
7240 * @param pVCpu The cross context virtual CPU structure.
7241 */
7242DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7243{
7244 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7245 {
7246 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7247 {
7248 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7249 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7250 AssertRC(rc);
7251 Log4(("Setup interrupt-window exiting\n"));
7252 }
7253 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7254}
7255
7256
7257/**
7258 * Clears the interrupt-window exiting control in the VMCS.
7259 *
7260 * @param pVCpu The cross context virtual CPU structure.
7261 */
7262DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7263{
7264 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7265 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7266 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7267 AssertRC(rc);
7268 Log4(("Cleared interrupt-window exiting\n"));
7269}
7270
7271
7272/**
7273 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7274 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7275 *
7276 * @param pVCpu The cross context virtual CPU structure.
7277 */
7278DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7279{
7280 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7281 {
7282 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7283 {
7284 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7285 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7286 AssertRC(rc);
7287 Log4(("Setup NMI-window exiting\n"));
7288 }
7289 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7290}
7291
7292
7293/**
7294 * Clears the NMI-window exiting control in the VMCS.
7295 *
7296 * @param pVCpu The cross context virtual CPU structure.
7297 */
7298DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7299{
7300 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7301 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7302 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7303 AssertRC(rc);
7304 Log4(("Cleared NMI-window exiting\n"));
7305}
7306
7307
7308/**
7309 * Evaluates the event to be delivered to the guest and sets it as the pending
7310 * event.
7311 *
7312 * @param pVCpu The cross context virtual CPU structure.
7313 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7314 * out-of-sync. Make sure to update the required fields
7315 * before using them.
7316 */
7317static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7318{
7319 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7320 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7321 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7322 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7323 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7324
7325 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7326 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7327 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7328 Assert(!TRPMHasTrap(pVCpu));
7329
7330 /*
7331 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7332 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7333 */
7334 /** @todo SMI. SMIs take priority over NMIs. */
7335 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7336 {
7337 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7338 if ( !pVCpu->hm.s.Event.fPending
7339 && !fBlockNmi
7340 && !fBlockSti
7341 && !fBlockMovSS)
7342 {
7343 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7344 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7345 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7346
7347 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7348 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7349 }
7350 else
7351 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7352 }
7353 /*
7354 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7355 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7356 */
7357 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7358 && !pVCpu->hm.s.fSingleInstruction)
7359 {
7360 Assert(!DBGFIsStepping(pVCpu));
7361 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7362 AssertRC(rc);
7363 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7364 if ( !pVCpu->hm.s.Event.fPending
7365 && !fBlockInt
7366 && !fBlockSti
7367 && !fBlockMovSS)
7368 {
7369 uint8_t u8Interrupt;
7370 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7371 if (RT_SUCCESS(rc))
7372 {
7373 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7374 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7375 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7376
7377 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7378 }
7379 else
7380 {
7381 /** @todo Does this actually happen? If not turn it into an assertion. */
7382 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7383 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7384 }
7385 }
7386 else
7387 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7388 }
7389}
7390
7391
7392/**
7393 * Sets a pending-debug exception to be delivered to the guest if the guest is
7394 * single-stepping in the VMCS.
7395 *
7396 * @param pVCpu The cross context virtual CPU structure.
7397 */
7398DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7399{
7400 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7401 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7402 AssertRC(rc);
7403}
7404
7405
7406/**
7407 * Injects any pending events into the guest if the guest is in a state to
7408 * receive them.
7409 *
7410 * @returns VBox status code (informational status codes included).
7411 * @param pVCpu The cross context virtual CPU structure.
7412 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7413 * out-of-sync. Make sure to update the required fields
7414 * before using them.
7415 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7416 * return VINF_EM_DBG_STEPPED if the event was
7417 * dispatched directly.
7418 */
7419static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7420{
7421 HMVMX_ASSERT_PREEMPT_SAFE();
7422 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7423
7424 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7425 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7426 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7427 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7428
7429 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7430 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7431 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7432 Assert(!TRPMHasTrap(pVCpu));
7433
7434 int rc = VINF_SUCCESS;
7435 if (pVCpu->hm.s.Event.fPending)
7436 {
7437 /*
7438 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7439 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7440 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7441 *
7442 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7443 */
7444 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7445#ifdef VBOX_STRICT
7446 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7447 {
7448 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7449 Assert(!fBlockInt);
7450 Assert(!fBlockSti);
7451 Assert(!fBlockMovSS);
7452 }
7453 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7454 {
7455 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7456 Assert(!fBlockSti);
7457 Assert(!fBlockMovSS);
7458 Assert(!fBlockNmi);
7459 }
7460#endif
7461 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7462 (uint8_t)uIntType));
7463 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7464 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7465 AssertRCReturn(rc, rc);
7466
7467 /* Update the interruptibility-state as it could have been changed by
7468 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7469 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7470 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7471
7472 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7473 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7474 else
7475 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7476 }
7477
7478 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7479 if ( fBlockSti
7480 || fBlockMovSS)
7481 {
7482 if (!pVCpu->hm.s.fSingleInstruction)
7483 {
7484 /*
7485 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7486 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7487 * See Intel spec. 27.3.4 "Saving Non-Register State".
7488 */
7489 Assert(!DBGFIsStepping(pVCpu));
7490 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7491 AssertRCReturn(rc2, rc2);
7492 if (pMixedCtx->eflags.Bits.u1TF)
7493 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7494 }
7495 else if (pMixedCtx->eflags.Bits.u1TF)
7496 {
7497 /*
7498 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7499 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7500 */
7501 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7502 uIntrState = 0;
7503 }
7504 }
7505
7506 /*
7507 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7508 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7509 */
7510 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7511 AssertRC(rc2);
7512
7513 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7514 NOREF(fBlockMovSS); NOREF(fBlockSti);
7515 return rc;
7516}
7517
7518
7519/**
7520 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7521 *
7522 * @param pVCpu The cross context virtual CPU structure.
7523 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7524 * out-of-sync. Make sure to update the required fields
7525 * before using them.
7526 */
7527DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7528{
7529 NOREF(pMixedCtx);
7530 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7531 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7532}
7533
7534
7535/**
7536 * Injects a double-fault (\#DF) exception into the VM.
7537 *
7538 * @returns VBox status code (informational status code included).
7539 * @param pVCpu The cross context virtual CPU structure.
7540 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7541 * out-of-sync. Make sure to update the required fields
7542 * before using them.
7543 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7544 * and should return VINF_EM_DBG_STEPPED if the event
7545 * is injected directly (register modified by us, not
7546 * by hardware on VM-entry).
7547 * @param puIntrState Pointer to the current guest interruptibility-state.
7548 * This interruptibility-state will be updated if
7549 * necessary. This cannot not be NULL.
7550 */
7551DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7552{
7553 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7554 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7555 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7556 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7557 fStepping, puIntrState);
7558}
7559
7560
7561/**
7562 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7563 *
7564 * @param pVCpu The cross context virtual CPU structure.
7565 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7566 * out-of-sync. Make sure to update the required fields
7567 * before using them.
7568 */
7569DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7570{
7571 NOREF(pMixedCtx);
7572 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7573 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7574 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7575}
7576
7577
7578/**
7579 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7580 *
7581 * @param pVCpu The cross context virtual CPU structure.
7582 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7583 * out-of-sync. Make sure to update the required fields
7584 * before using them.
7585 * @param cbInstr The value of RIP that is to be pushed on the guest
7586 * stack.
7587 */
7588DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7589{
7590 NOREF(pMixedCtx);
7591 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7592 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7593 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7594}
7595
7596
7597/**
7598 * Injects a general-protection (\#GP) fault into the VM.
7599 *
7600 * @returns VBox status code (informational status code included).
7601 * @param pVCpu The cross context virtual CPU structure.
7602 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7603 * out-of-sync. Make sure to update the required fields
7604 * before using them.
7605 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7606 * mode, i.e. in real-mode it's not valid).
7607 * @param u32ErrorCode The error code associated with the \#GP.
7608 * @param fStepping Whether we're running in
7609 * hmR0VmxRunGuestCodeStep() and should return
7610 * VINF_EM_DBG_STEPPED if the event is injected
7611 * directly (register modified by us, not by
7612 * hardware on VM-entry).
7613 * @param puIntrState Pointer to the current guest interruptibility-state.
7614 * This interruptibility-state will be updated if
7615 * necessary. This cannot not be NULL.
7616 */
7617DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7618 bool fStepping, uint32_t *puIntrState)
7619{
7620 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7621 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7622 if (fErrorCodeValid)
7623 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7624 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7625 fStepping, puIntrState);
7626}
7627
7628
7629/**
7630 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7631 * VM.
7632 *
7633 * @param pVCpu The cross context virtual CPU structure.
7634 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7635 * out-of-sync. Make sure to update the required fields
7636 * before using them.
7637 * @param u32ErrorCode The error code associated with the \#GP.
7638 */
7639DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7640{
7641 NOREF(pMixedCtx);
7642 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7643 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7644 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7645 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7646}
7647
7648
7649/**
7650 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7651 *
7652 * @param pVCpu The cross context virtual CPU structure.
7653 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7654 * out-of-sync. Make sure to update the required fields
7655 * before using them.
7656 * @param uVector The software interrupt vector number.
7657 * @param cbInstr The value of RIP that is to be pushed on the guest
7658 * stack.
7659 */
7660DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7661{
7662 NOREF(pMixedCtx);
7663 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7664 if ( uVector == X86_XCPT_BP
7665 || uVector == X86_XCPT_OF)
7666 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7667 else
7668 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7669 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7670}
7671
7672
7673/**
7674 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7675 * stack.
7676 *
7677 * @returns VBox status code (information status code included).
7678 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7679 * @param pVM The cross context VM structure.
7680 * @param pMixedCtx Pointer to the guest-CPU context.
7681 * @param uValue The value to push to the guest stack.
7682 */
7683DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7684{
7685 /*
7686 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7687 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7688 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7689 */
7690 if (pMixedCtx->sp == 1)
7691 return VINF_EM_RESET;
7692 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7693 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7694 AssertRCReturn(rc, rc);
7695 return rc;
7696}
7697
7698
7699/**
7700 * Injects an event into the guest upon VM-entry by updating the relevant fields
7701 * in the VM-entry area in the VMCS.
7702 *
7703 * @returns VBox status code (informational error codes included).
7704 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7705 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7706 *
7707 * @param pVCpu The cross context virtual CPU structure.
7708 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7709 * be out-of-sync. Make sure to update the required
7710 * fields before using them.
7711 * @param u64IntInfo The VM-entry interruption-information field.
7712 * @param cbInstr The VM-entry instruction length in bytes (for
7713 * software interrupts, exceptions and privileged
7714 * software exceptions).
7715 * @param u32ErrCode The VM-entry exception error code.
7716 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7717 * @param puIntrState Pointer to the current guest interruptibility-state.
7718 * This interruptibility-state will be updated if
7719 * necessary. This cannot not be NULL.
7720 * @param fStepping Whether we're running in
7721 * hmR0VmxRunGuestCodeStep() and should return
7722 * VINF_EM_DBG_STEPPED if the event is injected
7723 * directly (register modified by us, not by
7724 * hardware on VM-entry).
7725 *
7726 * @remarks Requires CR0!
7727 * @remarks No-long-jump zone!!!
7728 */
7729static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7730 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7731{
7732 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7733 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7734 Assert(puIntrState);
7735 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7736
7737 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7738 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7739
7740#ifdef VBOX_STRICT
7741 /* Validate the error-code-valid bit for hardware exceptions. */
7742 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7743 {
7744 switch (uVector)
7745 {
7746 case X86_XCPT_PF:
7747 case X86_XCPT_DF:
7748 case X86_XCPT_TS:
7749 case X86_XCPT_NP:
7750 case X86_XCPT_SS:
7751 case X86_XCPT_GP:
7752 case X86_XCPT_AC:
7753 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7754 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7755 /* fallthru */
7756 default:
7757 break;
7758 }
7759 }
7760#endif
7761
7762 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7763 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7764 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7765
7766 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7767
7768 /* We require CR0 to check if the guest is in real-mode. */
7769 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7770 AssertRCReturn(rc, rc);
7771
7772 /*
7773 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7774 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7775 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7776 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7777 */
7778 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7779 {
7780 PVM pVM = pVCpu->CTX_SUFF(pVM);
7781 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7782 {
7783 Assert(PDMVmmDevHeapIsEnabled(pVM));
7784 Assert(pVM->hm.s.vmx.pRealModeTSS);
7785
7786 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7787 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7788 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7789 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7790 AssertRCReturn(rc, rc);
7791 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7792
7793 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7794 size_t const cbIdtEntry = sizeof(X86IDTR16);
7795 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7796 {
7797 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7798 if (uVector == X86_XCPT_DF)
7799 return VINF_EM_RESET;
7800
7801 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7802 if (uVector == X86_XCPT_GP)
7803 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7804
7805 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7806 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7807 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7808 fStepping, puIntrState);
7809 }
7810
7811 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7812 uint16_t uGuestIp = pMixedCtx->ip;
7813 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7814 {
7815 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7816 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7817 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7818 }
7819 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7820 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7821
7822 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7823 X86IDTR16 IdtEntry;
7824 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7825 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7826 AssertRCReturn(rc, rc);
7827
7828 /* Construct the stack frame for the interrupt/exception handler. */
7829 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7830 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7831 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7832 AssertRCReturn(rc, rc);
7833
7834 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7835 if (rc == VINF_SUCCESS)
7836 {
7837 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7838 pMixedCtx->rip = IdtEntry.offSel;
7839 pMixedCtx->cs.Sel = IdtEntry.uSel;
7840 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7841 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7842 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7843 && uVector == X86_XCPT_PF)
7844 pMixedCtx->cr2 = GCPtrFaultAddress;
7845
7846 /* If any other guest-state bits are changed here, make sure to update
7847 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7848 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7849 | HM_CHANGED_GUEST_RIP
7850 | HM_CHANGED_GUEST_RFLAGS
7851 | HM_CHANGED_GUEST_RSP);
7852
7853 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7854 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7855 {
7856 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7857 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7858 Log4(("Clearing inhibition due to STI.\n"));
7859 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7860 }
7861 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7862 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7863
7864 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7865 it, if we are returning to ring-3 before executing guest code. */
7866 pVCpu->hm.s.Event.fPending = false;
7867
7868 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7869 if (fStepping)
7870 rc = VINF_EM_DBG_STEPPED;
7871 }
7872 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7873 return rc;
7874 }
7875
7876 /*
7877 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7878 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7879 */
7880 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7881 }
7882
7883 /* Validate. */
7884 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7885 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7886 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7887
7888 /* Inject. */
7889 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7890 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7891 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7892 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7893
7894 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7895 && uVector == X86_XCPT_PF)
7896 pMixedCtx->cr2 = GCPtrFaultAddress;
7897
7898 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7899 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7900
7901 AssertRCReturn(rc, rc);
7902 return rc;
7903}
7904
7905
7906/**
7907 * Clears the interrupt-window exiting control in the VMCS and if necessary
7908 * clears the current event in the VMCS as well.
7909 *
7910 * @returns VBox status code.
7911 * @param pVCpu The cross context virtual CPU structure.
7912 *
7913 * @remarks Use this function only to clear events that have not yet been
7914 * delivered to the guest but are injected in the VMCS!
7915 * @remarks No-long-jump zone!!!
7916 */
7917static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7918{
7919 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7920
7921 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7922 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7923
7924 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7925 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7926}
7927
7928
7929/**
7930 * Enters the VT-x session.
7931 *
7932 * @returns VBox status code.
7933 * @param pVM The cross context VM structure.
7934 * @param pVCpu The cross context virtual CPU structure.
7935 * @param pCpu Pointer to the CPU info struct.
7936 */
7937VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7938{
7939 AssertPtr(pVM);
7940 AssertPtr(pVCpu);
7941 Assert(pVM->hm.s.vmx.fSupported);
7942 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7943 NOREF(pCpu); NOREF(pVM);
7944
7945 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7946 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7947
7948#ifdef VBOX_STRICT
7949 /* Make sure we're in VMX root mode. */
7950 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7951 if (!(u32HostCR4 & X86_CR4_VMXE))
7952 {
7953 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7954 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7955 }
7956#endif
7957
7958 /*
7959 * Load the VCPU's VMCS as the current (and active) one.
7960 */
7961 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7962 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7963 if (RT_FAILURE(rc))
7964 return rc;
7965
7966 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7967 pVCpu->hm.s.fLeaveDone = false;
7968 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7969
7970 return VINF_SUCCESS;
7971}
7972
7973
7974/**
7975 * The thread-context callback (only on platforms which support it).
7976 *
7977 * @param enmEvent The thread-context event.
7978 * @param pVCpu The cross context virtual CPU structure.
7979 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7980 * @thread EMT(pVCpu)
7981 */
7982VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7983{
7984 NOREF(fGlobalInit);
7985
7986 switch (enmEvent)
7987 {
7988 case RTTHREADCTXEVENT_OUT:
7989 {
7990 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7991 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7992 VMCPU_ASSERT_EMT(pVCpu);
7993
7994 PVM pVM = pVCpu->CTX_SUFF(pVM);
7995 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7996
7997 /* No longjmps (logger flushes, locks) in this fragile context. */
7998 VMMRZCallRing3Disable(pVCpu);
7999 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8000
8001 /*
8002 * Restore host-state (FPU, debug etc.)
8003 */
8004 if (!pVCpu->hm.s.fLeaveDone)
8005 {
8006 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8007 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8008 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8009 pVCpu->hm.s.fLeaveDone = true;
8010 }
8011
8012 /* Leave HM context, takes care of local init (term). */
8013 int rc = HMR0LeaveCpu(pVCpu);
8014 AssertRC(rc); NOREF(rc);
8015
8016 /* Restore longjmp state. */
8017 VMMRZCallRing3Enable(pVCpu);
8018 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8019 break;
8020 }
8021
8022 case RTTHREADCTXEVENT_IN:
8023 {
8024 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8025 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8026 VMCPU_ASSERT_EMT(pVCpu);
8027
8028 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8029 VMMRZCallRing3Disable(pVCpu);
8030 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8031
8032 /* Initialize the bare minimum state required for HM. This takes care of
8033 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8034 int rc = HMR0EnterCpu(pVCpu);
8035 AssertRC(rc);
8036 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8037
8038 /* Load the active VMCS as the current one. */
8039 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8040 {
8041 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8042 AssertRC(rc); NOREF(rc);
8043 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8044 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8045 }
8046 pVCpu->hm.s.fLeaveDone = false;
8047
8048 /* Restore longjmp state. */
8049 VMMRZCallRing3Enable(pVCpu);
8050 break;
8051 }
8052
8053 default:
8054 break;
8055 }
8056}
8057
8058
8059/**
8060 * Saves the host state in the VMCS host-state.
8061 * Sets up the VM-exit MSR-load area.
8062 *
8063 * The CPU state will be loaded from these fields on every successful VM-exit.
8064 *
8065 * @returns VBox status code.
8066 * @param pVM The cross context VM structure.
8067 * @param pVCpu The cross context virtual CPU structure.
8068 *
8069 * @remarks No-long-jump zone!!!
8070 */
8071static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8072{
8073 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8074
8075 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8076 return VINF_SUCCESS;
8077
8078 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8079 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8080
8081 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8082 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8083
8084 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8085 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8086
8087 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8088 return rc;
8089}
8090
8091
8092/**
8093 * Saves the host state in the VMCS host-state.
8094 *
8095 * @returns VBox status code.
8096 * @param pVM The cross context VM structure.
8097 * @param pVCpu The cross context virtual CPU structure.
8098 *
8099 * @remarks No-long-jump zone!!!
8100 */
8101VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8102{
8103 AssertPtr(pVM);
8104 AssertPtr(pVCpu);
8105
8106 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8107
8108 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8109 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8110 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8111 return hmR0VmxSaveHostState(pVM, pVCpu);
8112}
8113
8114
8115/**
8116 * Loads the guest state into the VMCS guest-state area.
8117 *
8118 * The will typically be done before VM-entry when the guest-CPU state and the
8119 * VMCS state may potentially be out of sync.
8120 *
8121 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8122 * VM-entry controls.
8123 * Sets up the appropriate VMX non-root function to execute guest code based on
8124 * the guest CPU mode.
8125 *
8126 * @returns VBox status code.
8127 * @param pVM The cross context VM structure.
8128 * @param pVCpu The cross context virtual CPU structure.
8129 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8130 * out-of-sync. Make sure to update the required fields
8131 * before using them.
8132 *
8133 * @remarks No-long-jump zone!!!
8134 */
8135static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8136{
8137 AssertPtr(pVM);
8138 AssertPtr(pVCpu);
8139 AssertPtr(pMixedCtx);
8140 HMVMX_ASSERT_PREEMPT_SAFE();
8141
8142 VMMRZCallRing3Disable(pVCpu);
8143 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8144
8145 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8146
8147 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8148
8149 /* Determine real-on-v86 mode. */
8150 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8151 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8152 && CPUMIsGuestInRealModeEx(pMixedCtx))
8153 {
8154 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8155 }
8156
8157 /*
8158 * Load the guest-state into the VMCS.
8159 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8160 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8161 */
8162 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8163 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8164
8165 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8166 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8167 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8168
8169 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8170 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8171 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8172
8173 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8174 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8175
8176 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8177 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8178
8179 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8180 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8181 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8182
8183 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8184 determine we don't have to swap EFER after all. */
8185 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8186 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8187
8188 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8189 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8190
8191 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8192 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8193
8194 /*
8195 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8196 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8197 */
8198 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8199 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8200
8201 /* Clear any unused and reserved bits. */
8202 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8203
8204 VMMRZCallRing3Enable(pVCpu);
8205
8206 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8207 return rc;
8208}
8209
8210
8211/**
8212 * Loads the state shared between the host and guest into the VMCS.
8213 *
8214 * @param pVM The cross context VM structure.
8215 * @param pVCpu The cross context virtual CPU structure.
8216 * @param pCtx Pointer to the guest-CPU context.
8217 *
8218 * @remarks No-long-jump zone!!!
8219 */
8220static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8221{
8222 NOREF(pVM);
8223
8224 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8225 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8226
8227 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8228 {
8229 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8230 AssertRC(rc);
8231 }
8232
8233 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8234 {
8235 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8236 AssertRC(rc);
8237
8238 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8239 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8240 {
8241 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8242 AssertRC(rc);
8243 }
8244 }
8245
8246 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8247 {
8248#if HC_ARCH_BITS == 64
8249 if (pVM->hm.s.fAllow64BitGuests)
8250 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8251#endif
8252 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8253 }
8254
8255 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8256 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8257 {
8258 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8259 AssertRC(rc);
8260 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8261 }
8262
8263 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8264 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8265}
8266
8267
8268/**
8269 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8270 *
8271 * @param pVM The cross context VM structure.
8272 * @param pVCpu The cross context virtual CPU structure.
8273 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8274 * out-of-sync. Make sure to update the required fields
8275 * before using them.
8276 */
8277DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8278{
8279 HMVMX_ASSERT_PREEMPT_SAFE();
8280
8281 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8282#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8283 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8284#endif
8285
8286 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8287 {
8288 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8289 AssertRC(rc);
8290 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8291 }
8292 else if (HMCPU_CF_VALUE(pVCpu))
8293 {
8294 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8295 AssertRC(rc);
8296 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8297 }
8298
8299 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8300 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8301 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8302 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8303}
8304
8305
8306/**
8307 * Does the preparations before executing guest code in VT-x.
8308 *
8309 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8310 * recompiler/IEM. We must be cautious what we do here regarding committing
8311 * guest-state information into the VMCS assuming we assuredly execute the
8312 * guest in VT-x mode.
8313 *
8314 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8315 * the common-state (TRPM/forceflags), we must undo those changes so that the
8316 * recompiler/IEM can (and should) use them when it resumes guest execution.
8317 * Otherwise such operations must be done when we can no longer exit to ring-3.
8318 *
8319 * @returns Strict VBox status code.
8320 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8321 * have been disabled.
8322 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8323 * double-fault into the guest.
8324 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8325 * dispatched directly.
8326 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8327 *
8328 * @param pVM The cross context VM structure.
8329 * @param pVCpu The cross context virtual CPU structure.
8330 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8331 * out-of-sync. Make sure to update the required fields
8332 * before using them.
8333 * @param pVmxTransient Pointer to the VMX transient structure.
8334 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8335 * us ignore some of the reasons for returning to
8336 * ring-3, and return VINF_EM_DBG_STEPPED if event
8337 * dispatching took place.
8338 */
8339static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8340{
8341 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8342
8343#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8344 PGMRZDynMapFlushAutoSet(pVCpu);
8345#endif
8346
8347 /* Check force flag actions that might require us to go back to ring-3. */
8348 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8349 if (rc != VINF_SUCCESS)
8350 return rc;
8351
8352#ifndef IEM_VERIFICATION_MODE_FULL
8353 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8354 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8355 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8356 {
8357 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8358 RTGCPHYS GCPhysApicBase;
8359 GCPhysApicBase = pMixedCtx->msrApicBase;
8360 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8361
8362 /* Unalias any existing mapping. */
8363 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8364 AssertRCReturn(rc, rc);
8365
8366 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8367 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8368 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8369 AssertRCReturn(rc, rc);
8370
8371 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8372 }
8373#endif /* !IEM_VERIFICATION_MODE_FULL */
8374
8375 if (TRPMHasTrap(pVCpu))
8376 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8377 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8378
8379 /*
8380 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8381 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8382 */
8383 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8384 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8385 {
8386 Assert(rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
8387 return rc;
8388 }
8389
8390 /*
8391 * Load the guest state bits, we can handle longjmps/getting preempted here.
8392 *
8393 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8394 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8395 * Hence, this needs to be done -after- injection of events.
8396 */
8397 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8398
8399 /*
8400 * No longjmps to ring-3 from this point on!!!
8401 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8402 * This also disables flushing of the R0-logger instance (if any).
8403 */
8404 VMMRZCallRing3Disable(pVCpu);
8405
8406 /*
8407 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8408 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8409 *
8410 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8411 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8412 *
8413 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8414 * executing guest code.
8415 */
8416 pVmxTransient->fEFlags = ASMIntDisableFlags();
8417 if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8418 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8419 && ( !fStepping /* Optimized for the non-stepping case, of course. */
8420 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8421 {
8422 ASMSetFlags(pVmxTransient->fEFlags);
8423 VMMRZCallRing3Enable(pVCpu);
8424 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8425 return VINF_EM_RAW_TO_R3;
8426 }
8427
8428 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8429 {
8430 ASMSetFlags(pVmxTransient->fEFlags);
8431 VMMRZCallRing3Enable(pVCpu);
8432 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8433 return VINF_EM_RAW_INTERRUPT;
8434 }
8435
8436 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8437 pVCpu->hm.s.Event.fPending = false;
8438
8439 return VINF_SUCCESS;
8440}
8441
8442
8443/**
8444 * Prepares to run guest code in VT-x and we've committed to doing so. This
8445 * means there is no backing out to ring-3 or anywhere else at this
8446 * point.
8447 *
8448 * @param pVM The cross context VM structure.
8449 * @param pVCpu The cross context virtual CPU structure.
8450 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8451 * out-of-sync. Make sure to update the required fields
8452 * before using them.
8453 * @param pVmxTransient Pointer to the VMX transient structure.
8454 *
8455 * @remarks Called with preemption disabled.
8456 * @remarks No-long-jump zone!!!
8457 */
8458static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8459{
8460 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8461 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8462 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8463
8464 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8465 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8466
8467#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8468 if (!CPUMIsGuestFPUStateActive(pVCpu))
8469 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8470 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8471#endif
8472
8473 if ( pVCpu->hm.s.fPreloadGuestFpu
8474 && !CPUMIsGuestFPUStateActive(pVCpu))
8475 {
8476 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8477 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8478 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8479 }
8480
8481 /*
8482 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8483 */
8484 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8485 && pVCpu->hm.s.vmx.cMsrs > 0)
8486 {
8487 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8488 }
8489
8490 /*
8491 * Load the host state bits as we may've been preempted (only happens when
8492 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8493 */
8494 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8495 * any effect to the host state needing to be saved? */
8496 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8497 {
8498 /* This ASSUMES that pfnStartVM has been set up already. */
8499 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8500 AssertRC(rc);
8501 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8502 }
8503 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8504
8505 /*
8506 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8507 */
8508 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8509 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8510 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8511
8512 /* Store status of the shared guest-host state at the time of VM-entry. */
8513#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8514 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8515 {
8516 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8517 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8518 }
8519 else
8520#endif
8521 {
8522 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8523 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8524 }
8525 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8526
8527 /*
8528 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8529 */
8530 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8531 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8532
8533 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8534 RTCPUID idCurrentCpu = pCpu->idCpu;
8535 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8536 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8537 {
8538 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8539 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8540 }
8541
8542 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8543 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8544 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8545 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8546
8547 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8548
8549 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8550 to start executing. */
8551
8552 /*
8553 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8554 */
8555 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8556 {
8557 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8558 {
8559 bool fMsrUpdated;
8560 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8561 AssertRC(rc2);
8562 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8563
8564 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8565 &fMsrUpdated);
8566 AssertRC(rc2);
8567 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8568
8569 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8570 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8571 }
8572 else
8573 {
8574 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8575 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8576 }
8577 }
8578
8579#ifdef VBOX_STRICT
8580 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8581 hmR0VmxCheckHostEferMsr(pVCpu);
8582 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8583#endif
8584#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8585 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8586 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8587 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8588#endif
8589}
8590
8591
8592/**
8593 * Performs some essential restoration of state after running guest code in
8594 * VT-x.
8595 *
8596 * @param pVM The cross context VM structure.
8597 * @param pVCpu The cross context virtual CPU structure.
8598 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8599 * out-of-sync. Make sure to update the required fields
8600 * before using them.
8601 * @param pVmxTransient Pointer to the VMX transient structure.
8602 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8603 *
8604 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8605 *
8606 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8607 * unconditionally when it is safe to do so.
8608 */
8609static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8610{
8611 NOREF(pVM);
8612
8613 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8614
8615 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8616 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8617 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8618 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8619 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8620 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8621
8622 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8623 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8624
8625 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8626 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8627 Assert(!ASMIntAreEnabled());
8628 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8629
8630#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8631 if (CPUMIsGuestFPUStateActive(pVCpu))
8632 {
8633 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8634 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8635 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8636 }
8637#endif
8638
8639#if HC_ARCH_BITS == 64
8640 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8641#endif
8642 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8643#ifdef VBOX_STRICT
8644 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8645#endif
8646 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8647 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8648
8649 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8650 uint32_t uExitReason;
8651 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8652 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8653 AssertRC(rc);
8654 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8655 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8656
8657 /* Update the VM-exit history array. */
8658 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8659
8660 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8661 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8662 {
8663 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8664 pVmxTransient->fVMEntryFailed));
8665 return;
8666 }
8667
8668 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8669 {
8670 /** @todo We can optimize this by only syncing with our force-flags when
8671 * really needed and keeping the VMCS state as it is for most
8672 * VM-exits. */
8673 /* Update the guest interruptibility-state from the VMCS. */
8674 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8675
8676#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8677 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8678 AssertRC(rc);
8679#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8680 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8681 AssertRC(rc);
8682#endif
8683
8684 /*
8685 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8686 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8687 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8688 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8689 */
8690 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8691 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8692 {
8693 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8694 AssertRC(rc);
8695 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8696 }
8697 }
8698}
8699
8700
8701/**
8702 * Runs the guest code using VT-x the normal way.
8703 *
8704 * @returns VBox status code.
8705 * @param pVM The cross context VM structure.
8706 * @param pVCpu The cross context virtual CPU structure.
8707 * @param pCtx Pointer to the guest-CPU context.
8708 *
8709 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8710 */
8711static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8712{
8713 VMXTRANSIENT VmxTransient;
8714 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8715 int rc = VERR_INTERNAL_ERROR_5;
8716 uint32_t cLoops = 0;
8717
8718 for (;; cLoops++)
8719 {
8720 Assert(!HMR0SuspendPending());
8721 HMVMX_ASSERT_CPU_SAFE();
8722
8723 /* Preparatory work for running guest code, this may force us to return
8724 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8725 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8726 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8727 if (rc != VINF_SUCCESS)
8728 break;
8729
8730 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8731 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8732 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8733
8734 /* Restore any residual host-state and save any bits shared between host
8735 and guest into the guest-CPU state. Re-enables interrupts! */
8736 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8737
8738 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8739 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8740 {
8741 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8742 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8743 return rc;
8744 }
8745
8746 /* Profile the VM-exit. */
8747 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8748 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8749 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8750 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8751 HMVMX_START_EXIT_DISPATCH_PROF();
8752
8753 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8754 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8755 {
8756 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8757 hmR0VmxSaveGuestState(pVCpu, pCtx);
8758 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8759 }
8760
8761 /* Handle the VM-exit. */
8762#ifdef HMVMX_USE_FUNCTION_TABLE
8763 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8764#else
8765 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8766#endif
8767 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8768 if (rc != VINF_SUCCESS)
8769 break;
8770 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8771 {
8772 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8773 rc = VINF_EM_RAW_INTERRUPT;
8774 break;
8775 }
8776 }
8777
8778 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8779 return rc;
8780}
8781
8782
8783/**
8784 * Single steps guest code using VT-x.
8785 *
8786 * @returns VBox status code.
8787 * @param pVM The cross context VM structure.
8788 * @param pVCpu The cross context virtual CPU structure.
8789 * @param pCtx Pointer to the guest-CPU context.
8790 *
8791 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8792 */
8793static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8794{
8795 VMXTRANSIENT VmxTransient;
8796 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8797 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8798 uint32_t cLoops = 0;
8799 uint16_t uCsStart = pCtx->cs.Sel;
8800 uint64_t uRipStart = pCtx->rip;
8801
8802 for (;; cLoops++)
8803 {
8804 Assert(!HMR0SuspendPending());
8805 HMVMX_ASSERT_CPU_SAFE();
8806
8807 /* Preparatory work for running guest code, this may force us to return
8808 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8809 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8810 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8811 if (rcStrict != VINF_SUCCESS)
8812 break;
8813
8814 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8815 rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8816 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8817
8818 /* Restore any residual host-state and save any bits shared between host
8819 and guest into the guest-CPU state. Re-enables interrupts! */
8820 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8821
8822 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8823 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
8824 {
8825 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8826 hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
8827 return VBOXSTRICTRC_TODO(rcStrict);
8828 }
8829
8830 /* Profile the VM-exit. */
8831 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8832 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8833 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8834 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8835 HMVMX_START_EXIT_DISPATCH_PROF();
8836
8837 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8838 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8839 {
8840 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8841 hmR0VmxSaveGuestState(pVCpu, pCtx);
8842 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8843 }
8844
8845 /* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8846 rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8847 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8848 if (rcStrict != VINF_SUCCESS)
8849 break;
8850 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8851 {
8852 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8853 rcStrict = VINF_EM_RAW_INTERRUPT;
8854 break;
8855 }
8856
8857 /*
8858 * Did the RIP change, if so, consider it a single step.
8859 * Otherwise, make sure one of the TFs gets set.
8860 */
8861 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8862 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8863 AssertRCReturn(rc2, rc2);
8864 if ( pCtx->rip != uRipStart
8865 || pCtx->cs.Sel != uCsStart)
8866 {
8867 rcStrict = VINF_EM_DBG_STEPPED;
8868 break;
8869 }
8870 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8871 }
8872
8873 /*
8874 * Clear the X86_EFL_TF if necessary.
8875 */
8876 if (pVCpu->hm.s.fClearTrapFlag)
8877 {
8878 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8879 AssertRCReturn(rc2, rc2);
8880 pVCpu->hm.s.fClearTrapFlag = false;
8881 pCtx->eflags.Bits.u1TF = 0;
8882 }
8883 /** @todo there seems to be issues with the resume flag when the monitor trap
8884 * flag is pending without being used. Seen early in bios init when
8885 * accessing APIC page in protected mode. */
8886
8887 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8888 return VBOXSTRICTRC_TODO(rcStrict);
8889}
8890
8891
8892/**
8893 * Runs the guest code using VT-x.
8894 *
8895 * @returns VBox status code.
8896 * @param pVM The cross context VM structure.
8897 * @param pVCpu The cross context virtual CPU structure.
8898 * @param pCtx Pointer to the guest-CPU context.
8899 */
8900VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8901{
8902 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8903 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8904 HMVMX_ASSERT_PREEMPT_SAFE();
8905
8906 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8907
8908 int rc;
8909 if (!pVCpu->hm.s.fSingleInstruction)
8910 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8911 else
8912 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8913
8914 if (rc == VERR_EM_INTERPRETER)
8915 rc = VINF_EM_RAW_EMULATE_INSTR;
8916 else if (rc == VINF_EM_RESET)
8917 rc = VINF_EM_TRIPLE_FAULT;
8918
8919 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8920 if (RT_FAILURE(rc2))
8921 {
8922 pVCpu->hm.s.u32HMError = rc;
8923 rc = rc2;
8924 }
8925 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8926 return rc;
8927}
8928
8929
8930#ifndef HMVMX_USE_FUNCTION_TABLE
8931DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8932{
8933#ifdef DEBUG_ramshankar
8934# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8935# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8936#endif
8937 int rc;
8938 switch (rcReason)
8939 {
8940 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8941 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8942 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8943 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8944 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8945 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8946 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8947 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8948 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8949 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8950 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8951 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8952 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8953 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8954 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8955 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8956 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8957 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8958 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8959 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8960 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8961 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8962 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8963 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8964 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8965 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8966 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8967 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8968 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8969 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8970 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8971 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8972 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8973 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8974
8975 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8976 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8977 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8978 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8979 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8980 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8981 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8982 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8983 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8984
8985 case VMX_EXIT_VMCLEAR:
8986 case VMX_EXIT_VMLAUNCH:
8987 case VMX_EXIT_VMPTRLD:
8988 case VMX_EXIT_VMPTRST:
8989 case VMX_EXIT_VMREAD:
8990 case VMX_EXIT_VMRESUME:
8991 case VMX_EXIT_VMWRITE:
8992 case VMX_EXIT_VMXOFF:
8993 case VMX_EXIT_VMXON:
8994 case VMX_EXIT_INVEPT:
8995 case VMX_EXIT_INVVPID:
8996 case VMX_EXIT_VMFUNC:
8997 case VMX_EXIT_XSAVES:
8998 case VMX_EXIT_XRSTORS:
8999 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9000 break;
9001 case VMX_EXIT_RESERVED_60:
9002 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
9003 case VMX_EXIT_RESERVED_62:
9004 default:
9005 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9006 break;
9007 }
9008 return rc;
9009}
9010#endif /* !HMVMX_USE_FUNCTION_TABLE */
9011
9012
9013/**
9014 * Single-stepping VM-exit filtering.
9015 *
9016 * This is preprocessing the exits and deciding whether we've gotten far enough
9017 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9018 * performed.
9019 *
9020 * @returns Strict VBox status code.
9021 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9022 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9023 * out-of-sync. Make sure to update the required
9024 * fields before using them.
9025 * @param pVmxTransient Pointer to the VMX-transient structure.
9026 * @param uExitReason The VM-exit reason.
9027 * @param uCsStart The CS we started executing (stepping) on.
9028 * @param uRipStart The RIP we started executing (stepping) on.
9029 */
9030DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9031 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9032{
9033 switch (uExitReason)
9034 {
9035 case VMX_EXIT_XCPT_OR_NMI:
9036 {
9037 /* Check for host NMI. */
9038 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9039 AssertRCReturn(rc2, rc2);
9040 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9041 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9042 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9043 /* fall thru */
9044 }
9045
9046 case VMX_EXIT_EPT_MISCONFIG:
9047 case VMX_EXIT_TRIPLE_FAULT:
9048 case VMX_EXIT_APIC_ACCESS:
9049 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9050 case VMX_EXIT_TASK_SWITCH:
9051
9052 /* Instruction specific VM-exits: */
9053 case VMX_EXIT_IO_INSTR:
9054 case VMX_EXIT_CPUID:
9055 case VMX_EXIT_RDTSC:
9056 case VMX_EXIT_RDTSCP:
9057 case VMX_EXIT_MOV_CRX:
9058 case VMX_EXIT_MWAIT:
9059 case VMX_EXIT_MONITOR:
9060 case VMX_EXIT_RDMSR:
9061 case VMX_EXIT_WRMSR:
9062 case VMX_EXIT_MOV_DRX:
9063 case VMX_EXIT_HLT:
9064 case VMX_EXIT_INVD:
9065 case VMX_EXIT_INVLPG:
9066 case VMX_EXIT_RSM:
9067 case VMX_EXIT_PAUSE:
9068 case VMX_EXIT_XDTR_ACCESS:
9069 case VMX_EXIT_TR_ACCESS:
9070 case VMX_EXIT_WBINVD:
9071 case VMX_EXIT_XSETBV:
9072 case VMX_EXIT_RDRAND:
9073 case VMX_EXIT_INVPCID:
9074 case VMX_EXIT_GETSEC:
9075 case VMX_EXIT_RDPMC:
9076 case VMX_EXIT_VMCALL:
9077 case VMX_EXIT_VMCLEAR:
9078 case VMX_EXIT_VMLAUNCH:
9079 case VMX_EXIT_VMPTRLD:
9080 case VMX_EXIT_VMPTRST:
9081 case VMX_EXIT_VMREAD:
9082 case VMX_EXIT_VMRESUME:
9083 case VMX_EXIT_VMWRITE:
9084 case VMX_EXIT_VMXOFF:
9085 case VMX_EXIT_VMXON:
9086 case VMX_EXIT_INVEPT:
9087 case VMX_EXIT_INVVPID:
9088 case VMX_EXIT_VMFUNC:
9089 {
9090 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9091 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9092 AssertRCReturn(rc2, rc2);
9093 if ( pMixedCtx->rip != uRipStart
9094 || pMixedCtx->cs.Sel != uCsStart)
9095 return VINF_EM_DBG_STEPPED;
9096 break;
9097 }
9098 }
9099
9100 /*
9101 * Normal processing.
9102 */
9103#ifdef HMVMX_USE_FUNCTION_TABLE
9104 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9105#else
9106 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9107#endif
9108}
9109
9110
9111#ifdef VBOX_STRICT
9112/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9113# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9114 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9115
9116# define HMVMX_ASSERT_PREEMPT_CPUID() \
9117 do { \
9118 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9119 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9120 } while (0)
9121
9122# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9123 do { \
9124 AssertPtr(pVCpu); \
9125 AssertPtr(pMixedCtx); \
9126 AssertPtr(pVmxTransient); \
9127 Assert(pVmxTransient->fVMEntryFailed == false); \
9128 Assert(ASMIntAreEnabled()); \
9129 HMVMX_ASSERT_PREEMPT_SAFE(); \
9130 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9131 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9132 HMVMX_ASSERT_PREEMPT_SAFE(); \
9133 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9134 HMVMX_ASSERT_PREEMPT_CPUID(); \
9135 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9136 } while (0)
9137
9138# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9139 do { \
9140 Log4Func(("\n")); \
9141 } while (0)
9142#else /* nonstrict builds: */
9143# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9144 do { \
9145 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9146 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9147 } while (0)
9148# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9149#endif
9150
9151
9152/**
9153 * Advances the guest RIP after reading it from the VMCS.
9154 *
9155 * @returns VBox status code.
9156 * @param pVCpu The cross context virtual CPU structure.
9157 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9158 * out-of-sync. Make sure to update the required fields
9159 * before using them.
9160 * @param pVmxTransient Pointer to the VMX transient structure.
9161 *
9162 * @remarks No-long-jump zone!!!
9163 */
9164DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9165{
9166 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9167 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9168 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9169 AssertRCReturn(rc, rc);
9170
9171 pMixedCtx->rip += pVmxTransient->cbInstr;
9172 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9173
9174 /*
9175 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9176 * pending debug exception field as it takes care of priority of events.
9177 *
9178 * See Intel spec. 32.2.1 "Debug Exceptions".
9179 */
9180 if ( !pVCpu->hm.s.fSingleInstruction
9181 && pMixedCtx->eflags.Bits.u1TF)
9182 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
9183
9184 return rc;
9185}
9186
9187
9188/**
9189 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9190 * and update error record fields accordingly.
9191 *
9192 * @return VMX_IGS_* return codes.
9193 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9194 * wrong with the guest state.
9195 *
9196 * @param pVM The cross context VM structure.
9197 * @param pVCpu The cross context virtual CPU structure.
9198 * @param pCtx Pointer to the guest-CPU state.
9199 *
9200 * @remarks This function assumes our cache of the VMCS controls
9201 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9202 */
9203static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9204{
9205#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9206#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9207 uError = (err); \
9208 break; \
9209 } else do { } while (0)
9210
9211 int rc;
9212 uint32_t uError = VMX_IGS_ERROR;
9213 uint32_t u32Val;
9214 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9215
9216 do
9217 {
9218 /*
9219 * CR0.
9220 */
9221 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9222 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9223 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9224 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9225 if (fUnrestrictedGuest)
9226 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9227
9228 uint32_t u32GuestCR0;
9229 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9230 AssertRCBreak(rc);
9231 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9232 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9233 if ( !fUnrestrictedGuest
9234 && (u32GuestCR0 & X86_CR0_PG)
9235 && !(u32GuestCR0 & X86_CR0_PE))
9236 {
9237 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9238 }
9239
9240 /*
9241 * CR4.
9242 */
9243 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9244 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9245
9246 uint32_t u32GuestCR4;
9247 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9248 AssertRCBreak(rc);
9249 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9250 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9251
9252 /*
9253 * IA32_DEBUGCTL MSR.
9254 */
9255 uint64_t u64Val;
9256 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9257 AssertRCBreak(rc);
9258 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9259 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9260 {
9261 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9262 }
9263 uint64_t u64DebugCtlMsr = u64Val;
9264
9265#ifdef VBOX_STRICT
9266 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9267 AssertRCBreak(rc);
9268 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9269#endif
9270 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9271
9272 /*
9273 * RIP and RFLAGS.
9274 */
9275 uint32_t u32Eflags;
9276#if HC_ARCH_BITS == 64
9277 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9278 AssertRCBreak(rc);
9279 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9280 if ( !fLongModeGuest
9281 || !pCtx->cs.Attr.n.u1Long)
9282 {
9283 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9284 }
9285 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9286 * must be identical if the "IA-32e mode guest" VM-entry
9287 * control is 1 and CS.L is 1. No check applies if the
9288 * CPU supports 64 linear-address bits. */
9289
9290 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9291 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9292 AssertRCBreak(rc);
9293 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9294 VMX_IGS_RFLAGS_RESERVED);
9295 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9296 u32Eflags = u64Val;
9297#else
9298 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9299 AssertRCBreak(rc);
9300 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9301 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9302#endif
9303
9304 if ( fLongModeGuest
9305 || ( fUnrestrictedGuest
9306 && !(u32GuestCR0 & X86_CR0_PE)))
9307 {
9308 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9309 }
9310
9311 uint32_t u32EntryInfo;
9312 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9313 AssertRCBreak(rc);
9314 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9315 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9316 {
9317 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9318 }
9319
9320 /*
9321 * 64-bit checks.
9322 */
9323#if HC_ARCH_BITS == 64
9324 if (fLongModeGuest)
9325 {
9326 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9327 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9328 }
9329
9330 if ( !fLongModeGuest
9331 && (u32GuestCR4 & X86_CR4_PCIDE))
9332 {
9333 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9334 }
9335
9336 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9337 * 51:32 beyond the processor's physical-address width are 0. */
9338
9339 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9340 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9341 {
9342 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9343 }
9344
9345 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9346 AssertRCBreak(rc);
9347 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9348
9349 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9350 AssertRCBreak(rc);
9351 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9352#endif
9353
9354 /*
9355 * PERF_GLOBAL MSR.
9356 */
9357 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9358 {
9359 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9360 AssertRCBreak(rc);
9361 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9362 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9363 }
9364
9365 /*
9366 * PAT MSR.
9367 */
9368 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9369 {
9370 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9371 AssertRCBreak(rc);
9372 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9373 for (unsigned i = 0; i < 8; i++)
9374 {
9375 uint8_t u8Val = (u64Val & 0xff);
9376 if ( u8Val != 0 /* UC */
9377 && u8Val != 1 /* WC */
9378 && u8Val != 4 /* WT */
9379 && u8Val != 5 /* WP */
9380 && u8Val != 6 /* WB */
9381 && u8Val != 7 /* UC- */)
9382 {
9383 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9384 }
9385 u64Val >>= 8;
9386 }
9387 }
9388
9389 /*
9390 * EFER MSR.
9391 */
9392 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9393 {
9394 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9395 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9396 AssertRCBreak(rc);
9397 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9398 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9399 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
9400 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9401 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9402 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9403 || !(u32GuestCR0 & X86_CR0_PG)
9404 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9405 VMX_IGS_EFER_LMA_LME_MISMATCH);
9406 }
9407
9408 /*
9409 * Segment registers.
9410 */
9411 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9412 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9413 if (!(u32Eflags & X86_EFL_VM))
9414 {
9415 /* CS */
9416 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9417 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9418 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9419 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9420 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9421 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9422 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9423 /* CS cannot be loaded with NULL in protected mode. */
9424 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9425 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9426 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9427 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9428 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9429 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9430 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9431 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9432 else
9433 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9434
9435 /* SS */
9436 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9437 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9438 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9439 if ( !(pCtx->cr0 & X86_CR0_PE)
9440 || pCtx->cs.Attr.n.u4Type == 3)
9441 {
9442 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9443 }
9444 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9445 {
9446 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9447 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9448 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9449 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9450 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9451 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9452 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9453 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9454 }
9455
9456 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9457 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9458 {
9459 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9460 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9461 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9462 || pCtx->ds.Attr.n.u4Type > 11
9463 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9464 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9465 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9466 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9467 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9468 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9469 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9470 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9471 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9472 }
9473 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9474 {
9475 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9476 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9477 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9478 || pCtx->es.Attr.n.u4Type > 11
9479 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9480 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9481 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9482 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9483 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9484 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9485 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9486 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9487 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9488 }
9489 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9490 {
9491 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9492 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9493 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9494 || pCtx->fs.Attr.n.u4Type > 11
9495 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9496 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9497 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9498 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9499 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9500 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9501 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9502 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9503 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9504 }
9505 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9506 {
9507 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9508 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9509 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9510 || pCtx->gs.Attr.n.u4Type > 11
9511 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9512 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9513 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9514 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9515 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9516 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9517 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9518 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9519 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9520 }
9521 /* 64-bit capable CPUs. */
9522#if HC_ARCH_BITS == 64
9523 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9524 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9525 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9526 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9527 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9528 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9529 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9530 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9531 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9532 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9533 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9534#endif
9535 }
9536 else
9537 {
9538 /* V86 mode checks. */
9539 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9540 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9541 {
9542 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9543 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9544 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9545 }
9546 else
9547 {
9548 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9549 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9550 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9551 }
9552
9553 /* CS */
9554 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9555 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9556 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9557 /* SS */
9558 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9559 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9560 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9561 /* DS */
9562 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9563 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9564 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9565 /* ES */
9566 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9567 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9568 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9569 /* FS */
9570 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9571 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9572 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9573 /* GS */
9574 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9575 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9576 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9577 /* 64-bit capable CPUs. */
9578#if HC_ARCH_BITS == 64
9579 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9580 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9581 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9582 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9583 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9584 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9585 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9586 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9587 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9588 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9589 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9590#endif
9591 }
9592
9593 /*
9594 * TR.
9595 */
9596 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9597 /* 64-bit capable CPUs. */
9598#if HC_ARCH_BITS == 64
9599 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9600#endif
9601 if (fLongModeGuest)
9602 {
9603 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9604 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9605 }
9606 else
9607 {
9608 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9609 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9610 VMX_IGS_TR_ATTR_TYPE_INVALID);
9611 }
9612 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9613 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9614 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9615 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9616 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9617 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9618 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9619 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9620
9621 /*
9622 * GDTR and IDTR.
9623 */
9624#if HC_ARCH_BITS == 64
9625 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9626 AssertRCBreak(rc);
9627 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9628
9629 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9630 AssertRCBreak(rc);
9631 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9632#endif
9633
9634 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9635 AssertRCBreak(rc);
9636 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9637
9638 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9639 AssertRCBreak(rc);
9640 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9641
9642 /*
9643 * Guest Non-Register State.
9644 */
9645 /* Activity State. */
9646 uint32_t u32ActivityState;
9647 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9648 AssertRCBreak(rc);
9649 HMVMX_CHECK_BREAK( !u32ActivityState
9650 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9651 VMX_IGS_ACTIVITY_STATE_INVALID);
9652 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9653 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9654 uint32_t u32IntrState;
9655 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9656 AssertRCBreak(rc);
9657 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9658 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9659 {
9660 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9661 }
9662
9663 /** @todo Activity state and injecting interrupts. Left as a todo since we
9664 * currently don't use activity states but ACTIVE. */
9665
9666 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9667 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9668
9669 /* Guest interruptibility-state. */
9670 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9671 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9672 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9673 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9674 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9675 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9676 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9677 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9678 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9679 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9680 {
9681 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9682 {
9683 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9684 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9685 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9686 }
9687 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9688 {
9689 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9690 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9691 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9692 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9693 }
9694 }
9695 /** @todo Assumes the processor is not in SMM. */
9696 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9697 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9698 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9699 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9700 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9701 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9702 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9703 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9704 {
9705 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9706 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9707 }
9708
9709 /* Pending debug exceptions. */
9710#if HC_ARCH_BITS == 64
9711 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9712 AssertRCBreak(rc);
9713 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9714 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9715 u32Val = u64Val; /* For pending debug exceptions checks below. */
9716#else
9717 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9718 AssertRCBreak(rc);
9719 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9720 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9721#endif
9722
9723 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9724 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9725 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9726 {
9727 if ( (u32Eflags & X86_EFL_TF)
9728 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9729 {
9730 /* Bit 14 is PendingDebug.BS. */
9731 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9732 }
9733 if ( !(u32Eflags & X86_EFL_TF)
9734 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9735 {
9736 /* Bit 14 is PendingDebug.BS. */
9737 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9738 }
9739 }
9740
9741 /* VMCS link pointer. */
9742 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9743 AssertRCBreak(rc);
9744 if (u64Val != UINT64_C(0xffffffffffffffff))
9745 {
9746 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9747 /** @todo Bits beyond the processor's physical-address width MBZ. */
9748 /** @todo 32-bit located in memory referenced by value of this field (as a
9749 * physical address) must contain the processor's VMCS revision ID. */
9750 /** @todo SMM checks. */
9751 }
9752
9753 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9754 * not using Nested Paging? */
9755 if ( pVM->hm.s.fNestedPaging
9756 && !fLongModeGuest
9757 && CPUMIsGuestInPAEModeEx(pCtx))
9758 {
9759 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9760 AssertRCBreak(rc);
9761 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9762
9763 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9764 AssertRCBreak(rc);
9765 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9766
9767 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9768 AssertRCBreak(rc);
9769 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9770
9771 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9772 AssertRCBreak(rc);
9773 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9774 }
9775
9776 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9777 if (uError == VMX_IGS_ERROR)
9778 uError = VMX_IGS_REASON_NOT_FOUND;
9779 } while (0);
9780
9781 pVCpu->hm.s.u32HMError = uError;
9782 return uError;
9783
9784#undef HMVMX_ERROR_BREAK
9785#undef HMVMX_CHECK_BREAK
9786}
9787
9788/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9789/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9790/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9791
9792/** @name VM-exit handlers.
9793 * @{
9794 */
9795
9796/**
9797 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9798 */
9799HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9800{
9801 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9802 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9803 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9804 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
9805 return VINF_SUCCESS;
9806 return VINF_EM_RAW_INTERRUPT;
9807}
9808
9809
9810/**
9811 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9812 */
9813HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9814{
9815 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9816 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9817
9818 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9819 AssertRCReturn(rc, rc);
9820
9821 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9822 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9823 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9824 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9825
9826 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9827 {
9828 /*
9829 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9830 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9831 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9832 *
9833 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9834 */
9835 VMXDispatchHostNmi();
9836 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9837 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9838 return VINF_SUCCESS;
9839 }
9840
9841 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9842 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9843 if (RT_UNLIKELY(rc != VINF_SUCCESS))
9844 {
9845 if (rc == VINF_HM_DOUBLE_FAULT)
9846 rc = VINF_SUCCESS;
9847 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9848 return rc;
9849 }
9850
9851 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9852 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9853 switch (uIntType)
9854 {
9855 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9856 Assert(uVector == X86_XCPT_DB);
9857 /* no break */
9858 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9859 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9860 /* no break */
9861 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9862 {
9863 switch (uVector)
9864 {
9865 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9866 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9867 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9868 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9869 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9870 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9871#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9872 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9873 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9874 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9875 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9876 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9877 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9878 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9879 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9880 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9881 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9882 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9883 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9884#endif
9885 default:
9886 {
9887 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9888 AssertRCReturn(rc, rc);
9889
9890 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9891 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9892 {
9893 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9894 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9895 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9896
9897 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9898 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9899 AssertRCReturn(rc, rc);
9900 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9901 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9902 0 /* GCPtrFaultAddress */);
9903 AssertRCReturn(rc, rc);
9904 }
9905 else
9906 {
9907 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9908 pVCpu->hm.s.u32HMError = uVector;
9909 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9910 }
9911 break;
9912 }
9913 }
9914 break;
9915 }
9916
9917 default:
9918 {
9919 pVCpu->hm.s.u32HMError = uExitIntInfo;
9920 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9921 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9922 break;
9923 }
9924 }
9925 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9926 return rc;
9927}
9928
9929
9930/**
9931 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9932 */
9933HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9934{
9935 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9936
9937 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9938 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9939
9940 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9941 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9942 return VINF_SUCCESS;
9943}
9944
9945
9946/**
9947 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9948 */
9949HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9950{
9951 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9952 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
9953 {
9954 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9955 HMVMX_RETURN_UNEXPECTED_EXIT();
9956 }
9957
9958 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
9959
9960 /*
9961 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
9962 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
9963 */
9964 uint32_t uIntrState = 0;
9965 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9966 AssertRCReturn(rc, rc);
9967
9968 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
9969 if ( fBlockSti
9970 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
9971 {
9972 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
9973 }
9974
9975 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
9976 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
9977
9978 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9979 return VINF_SUCCESS;
9980}
9981
9982
9983/**
9984 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9985 */
9986HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9987{
9988 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9989 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9990 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9991}
9992
9993
9994/**
9995 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9996 */
9997HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9998{
9999 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10001 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10002}
10003
10004
10005/**
10006 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10007 */
10008HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10009{
10010 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10011 PVM pVM = pVCpu->CTX_SUFF(pVM);
10012 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10013 if (RT_LIKELY(rc == VINF_SUCCESS))
10014 {
10015 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10016 Assert(pVmxTransient->cbInstr == 2);
10017 }
10018 else
10019 {
10020 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10021 rc = VERR_EM_INTERPRETER;
10022 }
10023 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10024 return rc;
10025}
10026
10027
10028/**
10029 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10030 */
10031HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10032{
10033 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10034 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10035 AssertRCReturn(rc, rc);
10036
10037 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10038 return VINF_EM_RAW_EMULATE_INSTR;
10039
10040 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10041 HMVMX_RETURN_UNEXPECTED_EXIT();
10042}
10043
10044
10045/**
10046 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10047 */
10048HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10049{
10050 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10051 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10052 AssertRCReturn(rc, rc);
10053
10054 PVM pVM = pVCpu->CTX_SUFF(pVM);
10055 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10056 if (RT_LIKELY(rc == VINF_SUCCESS))
10057 {
10058 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10059 Assert(pVmxTransient->cbInstr == 2);
10060 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10061 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10062 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10063 }
10064 else
10065 rc = VERR_EM_INTERPRETER;
10066 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10067 return rc;
10068}
10069
10070
10071/**
10072 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10073 */
10074HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10075{
10076 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10077 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10078 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10079 AssertRCReturn(rc, rc);
10080
10081 PVM pVM = pVCpu->CTX_SUFF(pVM);
10082 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10083 if (RT_LIKELY(rc == VINF_SUCCESS))
10084 {
10085 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10086 Assert(pVmxTransient->cbInstr == 3);
10087 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10088 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10089 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10090 }
10091 else
10092 {
10093 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10094 rc = VERR_EM_INTERPRETER;
10095 }
10096 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10097 return rc;
10098}
10099
10100
10101/**
10102 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10103 */
10104HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10105{
10106 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10107 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10108 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10109 AssertRCReturn(rc, rc);
10110
10111 PVM pVM = pVCpu->CTX_SUFF(pVM);
10112 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10113 if (RT_LIKELY(rc == VINF_SUCCESS))
10114 {
10115 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10116 Assert(pVmxTransient->cbInstr == 2);
10117 }
10118 else
10119 {
10120 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10121 rc = VERR_EM_INTERPRETER;
10122 }
10123 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10124 return rc;
10125}
10126
10127
10128/**
10129 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10130 */
10131HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10132{
10133 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10134 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10135
10136 if (pVCpu->hm.s.fHypercallsEnabled)
10137 {
10138#if 0
10139 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10140 AssertRCReturn(rc, rc);
10141#else
10142 /* Aggressive state sync. for now. */
10143 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10144 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
10145#endif
10146 rc |= hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10147 AssertRCReturn(rc, rc);
10148
10149 rc = GIMHypercall(pVCpu, pMixedCtx);
10150 /* If the hypercall changes anything other than guest general-purpose registers,
10151 we would need to reload the guest changed bits here before VM-entry. */
10152 return rc;
10153 }
10154 else
10155 {
10156 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
10157 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10158 }
10159
10160 return VINF_SUCCESS;
10161}
10162
10163
10164/**
10165 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10166 */
10167HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10168{
10169 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10170 PVM pVM = pVCpu->CTX_SUFF(pVM);
10171 Assert(!pVM->hm.s.fNestedPaging);
10172
10173 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10174 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10175 AssertRCReturn(rc, rc);
10176
10177 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10178 rc = VBOXSTRICTRC_VAL(rc2);
10179 if (RT_LIKELY(rc == VINF_SUCCESS))
10180 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10181 else
10182 {
10183 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10184 pVmxTransient->uExitQualification, rc));
10185 }
10186 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10187 return rc;
10188}
10189
10190
10191/**
10192 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10193 */
10194HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10195{
10196 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10197 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10198 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10199 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10200 AssertRCReturn(rc, rc);
10201
10202 PVM pVM = pVCpu->CTX_SUFF(pVM);
10203 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10204 if (RT_LIKELY(rc == VINF_SUCCESS))
10205 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10206 else
10207 {
10208 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10209 rc = VERR_EM_INTERPRETER;
10210 }
10211 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10212 return rc;
10213}
10214
10215
10216/**
10217 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10218 */
10219HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10220{
10221 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10222 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10223 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10224 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10225 AssertRCReturn(rc, rc);
10226
10227 PVM pVM = pVCpu->CTX_SUFF(pVM);
10228 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10229 rc = VBOXSTRICTRC_VAL(rc2);
10230 if (RT_LIKELY( rc == VINF_SUCCESS
10231 || rc == VINF_EM_HALT))
10232 {
10233 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10234 AssertRCReturn(rc3, rc3);
10235
10236 if ( rc == VINF_EM_HALT
10237 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10238 {
10239 rc = VINF_SUCCESS;
10240 }
10241 }
10242 else
10243 {
10244 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10245 rc = VERR_EM_INTERPRETER;
10246 }
10247 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10248 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10249 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10250 return rc;
10251}
10252
10253
10254/**
10255 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10256 */
10257HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10258{
10259 /*
10260 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10261 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10262 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10263 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10264 */
10265 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10266 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10267 HMVMX_RETURN_UNEXPECTED_EXIT();
10268}
10269
10270
10271/**
10272 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10273 */
10274HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10275{
10276 /*
10277 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10278 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10279 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10280 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10281 */
10282 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10283 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10284 HMVMX_RETURN_UNEXPECTED_EXIT();
10285}
10286
10287
10288/**
10289 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10290 */
10291HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10292{
10293 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10294 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10295 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10296 HMVMX_RETURN_UNEXPECTED_EXIT();
10297}
10298
10299
10300/**
10301 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10302 */
10303HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10304{
10305 /*
10306 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10307 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10308 * See Intel spec. 25.3 "Other Causes of VM-exits".
10309 */
10310 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10311 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10312 HMVMX_RETURN_UNEXPECTED_EXIT();
10313}
10314
10315
10316/**
10317 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10318 * VM-exit.
10319 */
10320HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10321{
10322 /*
10323 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10324 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10325 *
10326 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10327 * See Intel spec. "23.8 Restrictions on VMX operation".
10328 */
10329 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10330 return VINF_SUCCESS;
10331}
10332
10333
10334/**
10335 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10336 * VM-exit.
10337 */
10338HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10339{
10340 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10341 return VINF_EM_RESET;
10342}
10343
10344
10345/**
10346 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10347 */
10348HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10349{
10350 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10351 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10352 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10353 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10354 AssertRCReturn(rc, rc);
10355
10356 pMixedCtx->rip++;
10357 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10358 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10359 rc = VINF_SUCCESS;
10360 else
10361 rc = VINF_EM_HALT;
10362
10363 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10364 if (rc != VINF_SUCCESS)
10365 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
10366 return rc;
10367}
10368
10369
10370/**
10371 * VM-exit handler for instructions that result in a \#UD exception delivered to
10372 * the guest.
10373 */
10374HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10375{
10376 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10377 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10378 return VINF_SUCCESS;
10379}
10380
10381
10382/**
10383 * VM-exit handler for expiry of the VMX preemption timer.
10384 */
10385HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10386{
10387 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10388
10389 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10390 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10391
10392 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10393 PVM pVM = pVCpu->CTX_SUFF(pVM);
10394 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10395 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10396 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10397}
10398
10399
10400/**
10401 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10402 */
10403HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10404{
10405 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10406
10407 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10408 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
10409 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10410 AssertRCReturn(rc, rc);
10411
10412 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
10413 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
10414
10415 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
10416
10417 return VBOXSTRICTRC_TODO(rcStrict);
10418}
10419
10420
10421/**
10422 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10423 */
10424HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10425{
10426 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10427
10428 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
10429 /** @todo implement EMInterpretInvpcid() */
10430 return VERR_EM_INTERPRETER;
10431}
10432
10433
10434/**
10435 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10436 * Error VM-exit.
10437 */
10438HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10439{
10440 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10441 AssertRCReturn(rc, rc);
10442
10443 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10444 AssertRCReturn(rc, rc);
10445
10446 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10447 NOREF(uInvalidReason);
10448
10449#ifdef VBOX_STRICT
10450 uint32_t uIntrState;
10451 RTHCUINTREG uHCReg;
10452 uint64_t u64Val;
10453 uint32_t u32Val;
10454
10455 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10456 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10457 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10458 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10459 AssertRCReturn(rc, rc);
10460
10461 Log4(("uInvalidReason %u\n", uInvalidReason));
10462 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10463 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10464 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10465 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10466
10467 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10468 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10469 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10470 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10471 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10472 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10473 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10474 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10475 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10476 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10477 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10478 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10479#else
10480 NOREF(pVmxTransient);
10481#endif
10482
10483 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10484 return VERR_VMX_INVALID_GUEST_STATE;
10485}
10486
10487
10488/**
10489 * VM-exit handler for VM-entry failure due to an MSR-load
10490 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10491 */
10492HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10493{
10494 NOREF(pVmxTransient);
10495 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10496 HMVMX_RETURN_UNEXPECTED_EXIT();
10497}
10498
10499
10500/**
10501 * VM-exit handler for VM-entry failure due to a machine-check event
10502 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10503 */
10504HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10505{
10506 NOREF(pVmxTransient);
10507 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10508 HMVMX_RETURN_UNEXPECTED_EXIT();
10509}
10510
10511
10512/**
10513 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10514 * theory.
10515 */
10516HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10517{
10518 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10519 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10520 return VERR_VMX_UNDEFINED_EXIT_CODE;
10521}
10522
10523
10524/**
10525 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10526 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10527 * Conditional VM-exit.
10528 */
10529HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10530{
10531 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10532
10533 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10534 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10535 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10536 return VERR_EM_INTERPRETER;
10537 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10538 HMVMX_RETURN_UNEXPECTED_EXIT();
10539}
10540
10541
10542/**
10543 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10544 */
10545HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10546{
10547 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10548
10549 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10550 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10551 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10552 return VERR_EM_INTERPRETER;
10553 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10554 HMVMX_RETURN_UNEXPECTED_EXIT();
10555}
10556
10557
10558/**
10559 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10560 */
10561HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10562{
10563 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10564
10565 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10566 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10567 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10568 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10569 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10570 {
10571 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10572 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10573 }
10574 AssertRCReturn(rc, rc);
10575 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10576
10577#ifdef VBOX_STRICT
10578 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10579 {
10580 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10581 && pMixedCtx->ecx != MSR_K6_EFER)
10582 {
10583 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10584 pMixedCtx->ecx));
10585 HMVMX_RETURN_UNEXPECTED_EXIT();
10586 }
10587# if HC_ARCH_BITS == 64
10588 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10589 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10590 {
10591 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10592 HMVMX_RETURN_UNEXPECTED_EXIT();
10593 }
10594# endif
10595 }
10596#endif
10597
10598 PVM pVM = pVCpu->CTX_SUFF(pVM);
10599 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10600 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10601 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10602 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10603 if (RT_LIKELY(rc == VINF_SUCCESS))
10604 {
10605 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10606 Assert(pVmxTransient->cbInstr == 2);
10607 }
10608 return rc;
10609}
10610
10611
10612/**
10613 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10614 */
10615HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10616{
10617 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10618 PVM pVM = pVCpu->CTX_SUFF(pVM);
10619 int rc = VINF_SUCCESS;
10620
10621 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10622 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10623 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10624 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10625 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10626 {
10627 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10628 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10629 }
10630 AssertRCReturn(rc, rc);
10631 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10632
10633 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10634 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10635 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10636
10637 if (RT_LIKELY(rc == VINF_SUCCESS))
10638 {
10639 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10640
10641 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10642 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10643 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10644 {
10645 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10646 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10647 EMInterpretWrmsr() changes it. */
10648 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10649 }
10650 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10651 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10652 else if (pMixedCtx->ecx == MSR_K6_EFER)
10653 {
10654 /*
10655 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10656 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10657 * the other bits as well, SCE and NXE. See @bugref{7368}.
10658 */
10659 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10660 }
10661
10662 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10663 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10664 {
10665 switch (pMixedCtx->ecx)
10666 {
10667 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10668 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10669 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10670 case MSR_K8_FS_BASE: /* no break */
10671 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10672 case MSR_K6_EFER: /* already handled above */ break;
10673 default:
10674 {
10675 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10676 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10677#if HC_ARCH_BITS == 64
10678 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10679 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10680#endif
10681 break;
10682 }
10683 }
10684 }
10685#ifdef VBOX_STRICT
10686 else
10687 {
10688 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10689 switch (pMixedCtx->ecx)
10690 {
10691 case MSR_IA32_SYSENTER_CS:
10692 case MSR_IA32_SYSENTER_EIP:
10693 case MSR_IA32_SYSENTER_ESP:
10694 case MSR_K8_FS_BASE:
10695 case MSR_K8_GS_BASE:
10696 {
10697 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10698 HMVMX_RETURN_UNEXPECTED_EXIT();
10699 }
10700
10701 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10702 default:
10703 {
10704 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10705 {
10706 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10707 if (pMixedCtx->ecx != MSR_K6_EFER)
10708 {
10709 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10710 pMixedCtx->ecx));
10711 HMVMX_RETURN_UNEXPECTED_EXIT();
10712 }
10713 }
10714
10715#if HC_ARCH_BITS == 64
10716 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10717 {
10718 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10719 HMVMX_RETURN_UNEXPECTED_EXIT();
10720 }
10721#endif
10722 break;
10723 }
10724 }
10725 }
10726#endif /* VBOX_STRICT */
10727 }
10728 return rc;
10729}
10730
10731
10732/**
10733 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10734 */
10735HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10736{
10737 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10738
10739 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10740 return VINF_EM_RAW_INTERRUPT;
10741}
10742
10743
10744/**
10745 * VM-exit handler for when the TPR value is lowered below the specified
10746 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10747 */
10748HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10749{
10750 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10751 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10752
10753 /*
10754 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10755 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10756 * resume guest execution.
10757 */
10758 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10759 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10760 return VINF_SUCCESS;
10761}
10762
10763
10764/**
10765 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10766 * VM-exit.
10767 *
10768 * @retval VINF_SUCCESS when guest execution can continue.
10769 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10770 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10771 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10772 * interpreter.
10773 */
10774HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10775{
10776 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10777 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10778 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10779 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10780 AssertRCReturn(rc, rc);
10781
10782 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10783 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10784 PVM pVM = pVCpu->CTX_SUFF(pVM);
10785 VBOXSTRICTRC rcStrict;
10786 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
10787 switch (uAccessType)
10788 {
10789 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10790 {
10791 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10792 AssertRCReturn(rc, rc);
10793
10794 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
10795 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10796 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10797 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
10798 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10799 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10800 {
10801 case 0: /* CR0 */
10802 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10803 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
10804 break;
10805 case 2: /* CR2 */
10806 /* Nothing to do here, CR2 it's not part of the VMCS. */
10807 break;
10808 case 3: /* CR3 */
10809 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10810 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10811 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
10812 break;
10813 case 4: /* CR4 */
10814 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10815 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
10816 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
10817 break;
10818 case 8: /* CR8 */
10819 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10820 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
10821 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10822 break;
10823 default:
10824 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10825 break;
10826 }
10827
10828 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10829 break;
10830 }
10831
10832 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10833 {
10834 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10835 AssertRCReturn(rc, rc);
10836
10837 Assert( !pVM->hm.s.fNestedPaging
10838 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10839 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10840
10841 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10842 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10843 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10844
10845 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
10846 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10847 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10848 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10849 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10850 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10851 VBOXSTRICTRC_VAL(rcStrict)));
10852 break;
10853 }
10854
10855 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10856 {
10857 AssertRCReturn(rc, rc);
10858 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
10859 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10860 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10861 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10862 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
10863 break;
10864 }
10865
10866 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10867 {
10868 AssertRCReturn(rc, rc);
10869 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
10870 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10871 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
10872 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10873 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10874 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
10875 break;
10876 }
10877
10878 default:
10879 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
10880 VERR_VMX_UNEXPECTED_EXCEPTION);
10881 }
10882
10883 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
10884 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10885 NOREF(pVM);
10886 return VBOXSTRICTRC_TODO(rcStrict);
10887}
10888
10889
10890/**
10891 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10892 * VM-exit.
10893 */
10894HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10895{
10896 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10897 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10898
10899 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10900 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10901 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10902 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10903 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10904 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10905 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10906 AssertRCReturn(rc2, rc2);
10907
10908 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10909 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10910 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10911 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10912 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10913 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10914 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10915 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
10916 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10917
10918 /* I/O operation lookup arrays. */
10919 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10920 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10921
10922 VBOXSTRICTRC rcStrict;
10923 uint32_t const cbValue = s_aIOSizes[uIOWidth];
10924 uint32_t const cbInstr = pVmxTransient->cbInstr;
10925 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10926 PVM pVM = pVCpu->CTX_SUFF(pVM);
10927 if (fIOString)
10928 {
10929#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
10930 See @bugref{5752#c158}. Should work now. */
10931 /*
10932 * INS/OUTS - I/O String instruction.
10933 *
10934 * Use instruction-information if available, otherwise fall back on
10935 * interpreting the instruction.
10936 */
10937 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
10938 fIOWrite ? 'w' : 'r'));
10939 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
10940 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10941 {
10942 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10943 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10944 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10945 AssertRCReturn(rc2, rc2);
10946 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
10947 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10948 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10949 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10950 if (fIOWrite)
10951 {
10952 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10953 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10954 }
10955 else
10956 {
10957 /*
10958 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10959 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10960 * See Intel Instruction spec. for "INS".
10961 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10962 */
10963 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10964 }
10965 }
10966 else
10967 {
10968 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10969 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10970 AssertRCReturn(rc2, rc2);
10971 rcStrict = IEMExecOne(pVCpu);
10972 }
10973 /** @todo IEM needs to be setting these flags somehow. */
10974 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10975 fUpdateRipAlready = true;
10976#else
10977 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10978 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10979 if (RT_SUCCESS(rcStrict))
10980 {
10981 if (fIOWrite)
10982 {
10983 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10984 (DISCPUMODE)pDis->uAddrMode, cbValue);
10985 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10986 }
10987 else
10988 {
10989 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10990 (DISCPUMODE)pDis->uAddrMode, cbValue);
10991 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10992 }
10993 }
10994 else
10995 {
10996 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
10997 pMixedCtx->rip));
10998 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10999 }
11000#endif
11001 }
11002 else
11003 {
11004 /*
11005 * IN/OUT - I/O instruction.
11006 */
11007 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11008 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11009 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11010 if (fIOWrite)
11011 {
11012 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11013 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11014 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11015 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11016 }
11017 else
11018 {
11019 uint32_t u32Result = 0;
11020 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11021 if (IOM_SUCCESS(rcStrict))
11022 {
11023 /* Save result of I/O IN instr. in AL/AX/EAX. */
11024 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11025 }
11026 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11027 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11028 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11029 }
11030 }
11031
11032 if (IOM_SUCCESS(rcStrict))
11033 {
11034 if (!fUpdateRipAlready)
11035 {
11036 pMixedCtx->rip += cbInstr;
11037 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11038 }
11039
11040 /*
11041 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11042 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11043 */
11044 if (fIOString)
11045 {
11046 /** @todo Single-step for INS/OUTS with REP prefix? */
11047 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11048 }
11049 else if ( !fDbgStepping
11050 && fGstStepping)
11051 {
11052 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
11053 }
11054
11055 /*
11056 * If any I/O breakpoints are armed, we need to check if one triggered
11057 * and take appropriate action.
11058 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11059 */
11060 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11061 AssertRCReturn(rc2, rc2);
11062
11063 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11064 * execution engines about whether hyper BPs and such are pending. */
11065 uint32_t const uDr7 = pMixedCtx->dr[7];
11066 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11067 && X86_DR7_ANY_RW_IO(uDr7)
11068 && (pMixedCtx->cr4 & X86_CR4_DE))
11069 || DBGFBpIsHwIoArmed(pVM)))
11070 {
11071 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11072
11073 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11074 VMMRZCallRing3Disable(pVCpu);
11075 HM_DISABLE_PREEMPT();
11076
11077 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11078
11079 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11080 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11081 {
11082 /* Raise #DB. */
11083 if (fIsGuestDbgActive)
11084 ASMSetDR6(pMixedCtx->dr[6]);
11085 if (pMixedCtx->dr[7] != uDr7)
11086 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11087
11088 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11089 }
11090 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11091 else if ( rcStrict2 != VINF_SUCCESS
11092 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11093 rcStrict = rcStrict2;
11094
11095 HM_RESTORE_PREEMPT();
11096 VMMRZCallRing3Enable(pVCpu);
11097 }
11098 }
11099
11100#ifdef VBOX_STRICT
11101 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11102 Assert(!fIOWrite);
11103 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11104 Assert(fIOWrite);
11105 else
11106 {
11107#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11108 * statuses, that the VMM device and some others may return. See
11109 * IOM_SUCCESS() for guidance. */
11110 AssertMsg( RT_FAILURE(rcStrict)
11111 || rcStrict == VINF_SUCCESS
11112 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11113 || rcStrict == VINF_EM_DBG_BREAKPOINT
11114 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11115 || rcStrict == VINF_EM_RAW_TO_R3
11116 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11117#endif
11118 }
11119#endif
11120
11121 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11122 return VBOXSTRICTRC_TODO(rcStrict);
11123}
11124
11125
11126/**
11127 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11128 * VM-exit.
11129 */
11130HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11131{
11132 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11133
11134 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11135 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11136 AssertRCReturn(rc, rc);
11137 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11138 {
11139 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11140 AssertRCReturn(rc, rc);
11141 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11142 {
11143 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11144
11145 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11146 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11147
11148 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11149 Assert(!pVCpu->hm.s.Event.fPending);
11150 pVCpu->hm.s.Event.fPending = true;
11151 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11152 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11153 AssertRCReturn(rc, rc);
11154 if (fErrorCodeValid)
11155 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11156 else
11157 pVCpu->hm.s.Event.u32ErrCode = 0;
11158 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11159 && uVector == X86_XCPT_PF)
11160 {
11161 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11162 }
11163
11164 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11165 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11166 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11167 }
11168 }
11169
11170 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11171 * emulation. */
11172 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11173 return VERR_EM_INTERPRETER;
11174}
11175
11176
11177/**
11178 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11179 */
11180HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11181{
11182 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11183 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11184 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11185 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11186 AssertRCReturn(rc, rc);
11187 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11188 return VINF_EM_DBG_STEPPED;
11189}
11190
11191
11192/**
11193 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11194 */
11195HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11196{
11197 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11198
11199 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11200 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11201 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11202 {
11203 if (rc == VINF_HM_DOUBLE_FAULT)
11204 rc = VINF_SUCCESS;
11205 return rc;
11206 }
11207
11208#if 0
11209 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11210 * just sync the whole thing. */
11211 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11212#else
11213 /* Aggressive state sync. for now. */
11214 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11215 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11216 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11217#endif
11218 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11219 AssertRCReturn(rc, rc);
11220
11221 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11222 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11223 switch (uAccessType)
11224 {
11225 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11226 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11227 {
11228 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11229 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11230 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11231
11232 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11233 GCPhys &= PAGE_BASE_GC_MASK;
11234 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11235 PVM pVM = pVCpu->CTX_SUFF(pVM);
11236 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11237 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11238
11239 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11240 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11241 CPUMCTX2CORE(pMixedCtx), GCPhys);
11242 rc = VBOXSTRICTRC_VAL(rc2);
11243 Log4(("ApicAccess rc=%d\n", rc));
11244 if ( rc == VINF_SUCCESS
11245 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11246 || rc == VERR_PAGE_NOT_PRESENT)
11247 {
11248 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11249 | HM_CHANGED_GUEST_RSP
11250 | HM_CHANGED_GUEST_RFLAGS
11251 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11252 rc = VINF_SUCCESS;
11253 }
11254 break;
11255 }
11256
11257 default:
11258 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11259 rc = VINF_EM_RAW_EMULATE_INSTR;
11260 break;
11261 }
11262
11263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11264 if (rc != VINF_SUCCESS)
11265 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
11266 return rc;
11267}
11268
11269
11270/**
11271 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11272 * VM-exit.
11273 */
11274HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11275{
11276 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11277
11278 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11279 if (pVmxTransient->fWasGuestDebugStateActive)
11280 {
11281 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11282 HMVMX_RETURN_UNEXPECTED_EXIT();
11283 }
11284
11285 int rc = VERR_INTERNAL_ERROR_5;
11286 if ( !pVCpu->hm.s.fSingleInstruction
11287 && !pVmxTransient->fWasHyperDebugStateActive)
11288 {
11289 Assert(!DBGFIsStepping(pVCpu));
11290
11291 /* Don't intercept MOV DRx and #DB any more. */
11292 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11293 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11294 AssertRCReturn(rc, rc);
11295
11296 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11297 {
11298#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11299 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11300 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
11301#endif
11302 }
11303
11304 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11305 VMMRZCallRing3Disable(pVCpu);
11306 HM_DISABLE_PREEMPT();
11307
11308 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11309 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11310 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11311
11312 HM_RESTORE_PREEMPT();
11313 VMMRZCallRing3Enable(pVCpu);
11314
11315#ifdef VBOX_WITH_STATISTICS
11316 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11317 AssertRCReturn(rc, rc);
11318 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11319 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11320 else
11321 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11322#endif
11323 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11324 return VINF_SUCCESS;
11325 }
11326
11327 /*
11328 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11329 * Update the segment registers and DR7 from the CPU.
11330 */
11331 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11332 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11333 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11334 AssertRCReturn(rc, rc);
11335 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11336
11337 PVM pVM = pVCpu->CTX_SUFF(pVM);
11338 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11339 {
11340 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11341 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11342 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11343 if (RT_SUCCESS(rc))
11344 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11345 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11346 }
11347 else
11348 {
11349 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11350 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11351 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11352 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11353 }
11354
11355 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11356 if (RT_SUCCESS(rc))
11357 {
11358 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11359 AssertRCReturn(rc2, rc2);
11360 }
11361 return rc;
11362}
11363
11364
11365/**
11366 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11367 * Conditional VM-exit.
11368 */
11369HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11370{
11371 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11372 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11373
11374 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11375 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11376 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11377 {
11378 if (rc == VINF_HM_DOUBLE_FAULT)
11379 rc = VINF_SUCCESS;
11380 return rc;
11381 }
11382
11383 RTGCPHYS GCPhys = 0;
11384 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11385
11386#if 0
11387 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11388#else
11389 /* Aggressive state sync. for now. */
11390 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11391 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11392 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11393#endif
11394 AssertRCReturn(rc, rc);
11395
11396 /*
11397 * If we succeed, resume guest execution.
11398 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11399 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11400 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11401 * weird case. See @bugref{6043}.
11402 */
11403 PVM pVM = pVCpu->CTX_SUFF(pVM);
11404 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11405 rc = VBOXSTRICTRC_VAL(rc2);
11406 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11407 if ( rc == VINF_SUCCESS
11408 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11409 || rc == VERR_PAGE_NOT_PRESENT)
11410 {
11411 /* Successfully handled MMIO operation. */
11412 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11413 | HM_CHANGED_GUEST_RSP
11414 | HM_CHANGED_GUEST_RFLAGS
11415 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11416 rc = VINF_SUCCESS;
11417 }
11418 return rc;
11419}
11420
11421
11422/**
11423 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11424 * VM-exit.
11425 */
11426HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11427{
11428 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11429 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11430
11431 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11432 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11433 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11434 {
11435 if (rc == VINF_HM_DOUBLE_FAULT)
11436 rc = VINF_SUCCESS;
11437 return rc;
11438 }
11439
11440 RTGCPHYS GCPhys = 0;
11441 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11442 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11443#if 0
11444 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11445#else
11446 /* Aggressive state sync. for now. */
11447 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11448 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11449 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11450#endif
11451 AssertRCReturn(rc, rc);
11452
11453 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11454 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11455
11456 RTGCUINT uErrorCode = 0;
11457 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11458 uErrorCode |= X86_TRAP_PF_ID;
11459 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11460 uErrorCode |= X86_TRAP_PF_RW;
11461 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11462 uErrorCode |= X86_TRAP_PF_P;
11463
11464 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11465
11466 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11467 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11468
11469 /* Handle the pagefault trap for the nested shadow table. */
11470 PVM pVM = pVCpu->CTX_SUFF(pVM);
11471 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11472 TRPMResetTrap(pVCpu);
11473
11474 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11475 if ( rc == VINF_SUCCESS
11476 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11477 || rc == VERR_PAGE_NOT_PRESENT)
11478 {
11479 /* Successfully synced our nested page tables. */
11480 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11481 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11482 | HM_CHANGED_GUEST_RSP
11483 | HM_CHANGED_GUEST_RFLAGS);
11484 return VINF_SUCCESS;
11485 }
11486
11487 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11488 return rc;
11489}
11490
11491/** @} */
11492
11493/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11494/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11495/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11496
11497/** @name VM-exit exception handlers.
11498 * @{
11499 */
11500
11501/**
11502 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
11503 */
11504static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11505{
11506 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11507 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11508
11509 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11510 AssertRCReturn(rc, rc);
11511
11512 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11513 {
11514 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11515 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11516
11517 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11518 * provides VM-exit instruction length. If this causes problem later,
11519 * disassemble the instruction like it's done on AMD-V. */
11520 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11521 AssertRCReturn(rc2, rc2);
11522 return rc;
11523 }
11524
11525 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11526 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11527 return rc;
11528}
11529
11530
11531/**
11532 * VM-exit exception handler for \#BP (Breakpoint exception).
11533 */
11534static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11535{
11536 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11537 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11538
11539 /** @todo Try optimize this by not saving the entire guest state unless
11540 * really needed. */
11541 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11542 AssertRCReturn(rc, rc);
11543
11544 PVM pVM = pVCpu->CTX_SUFF(pVM);
11545 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11546 if (rc == VINF_EM_RAW_GUEST_TRAP)
11547 {
11548 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11549 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11550 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11551 AssertRCReturn(rc, rc);
11552
11553 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11554 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11555 }
11556
11557 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11558 return rc;
11559}
11560
11561
11562/**
11563 * VM-exit exception handler for \#DB (Debug exception).
11564 */
11565static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11566{
11567 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11568 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11569 Log6(("XcptDB\n"));
11570
11571 /*
11572 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11573 * for processing.
11574 */
11575 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11576 AssertRCReturn(rc, rc);
11577
11578 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11579 uint64_t uDR6 = X86_DR6_INIT_VAL;
11580 uDR6 |= ( pVmxTransient->uExitQualification
11581 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11582
11583 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11584 if (rc == VINF_EM_RAW_GUEST_TRAP)
11585 {
11586 /*
11587 * The exception was for the guest. Update DR6, DR7.GD and
11588 * IA32_DEBUGCTL.LBR before forwarding it.
11589 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11590 */
11591 VMMRZCallRing3Disable(pVCpu);
11592 HM_DISABLE_PREEMPT();
11593
11594 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11595 pMixedCtx->dr[6] |= uDR6;
11596 if (CPUMIsGuestDebugStateActive(pVCpu))
11597 ASMSetDR6(pMixedCtx->dr[6]);
11598
11599 HM_RESTORE_PREEMPT();
11600 VMMRZCallRing3Enable(pVCpu);
11601
11602 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11603 AssertRCReturn(rc, rc);
11604
11605 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11606 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11607
11608 /* Paranoia. */
11609 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11610 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11611
11612 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11613 AssertRCReturn(rc, rc);
11614
11615 /*
11616 * Raise #DB in the guest.
11617 *
11618 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11619 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11620 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11621 *
11622 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11623 */
11624 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11625 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11626 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11627 AssertRCReturn(rc, rc);
11628 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11629 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11630 return VINF_SUCCESS;
11631 }
11632
11633 /*
11634 * Not a guest trap, must be a hypervisor related debug event then.
11635 * Update DR6 in case someone is interested in it.
11636 */
11637 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11638 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11639 CPUMSetHyperDR6(pVCpu, uDR6);
11640
11641 return rc;
11642}
11643
11644
11645/**
11646 * VM-exit exception handler for \#NM (Device-not-available exception: floating
11647 * point exception).
11648 */
11649static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11650{
11651 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11652
11653 /* We require CR0 and EFER. EFER is always up-to-date. */
11654 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11655 AssertRCReturn(rc, rc);
11656
11657 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11658 VMMRZCallRing3Disable(pVCpu);
11659 HM_DISABLE_PREEMPT();
11660
11661 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11662 if (pVmxTransient->fWasGuestFPUStateActive)
11663 {
11664 rc = VINF_EM_RAW_GUEST_TRAP;
11665 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11666 }
11667 else
11668 {
11669#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11670 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11671#endif
11672 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11673 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11674 }
11675
11676 HM_RESTORE_PREEMPT();
11677 VMMRZCallRing3Enable(pVCpu);
11678
11679 if (rc == VINF_SUCCESS)
11680 {
11681 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11682 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11683 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11684 pVCpu->hm.s.fPreloadGuestFpu = true;
11685 }
11686 else
11687 {
11688 /* Forward #NM to the guest. */
11689 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11690 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11691 AssertRCReturn(rc, rc);
11692 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11693 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11694 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11695 }
11696
11697 return VINF_SUCCESS;
11698}
11699
11700
11701/**
11702 * VM-exit exception handler for \#GP (General-protection exception).
11703 *
11704 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11705 */
11706static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11707{
11708 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11709 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11710
11711 int rc = VERR_INTERNAL_ERROR_5;
11712 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11713 {
11714#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11715 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11716 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11717 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11718 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11719 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11720 AssertRCReturn(rc, rc);
11721 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11722 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11723 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11724 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11725 return rc;
11726#else
11727 /* We don't intercept #GP. */
11728 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11729 NOREF(pVmxTransient);
11730 return VERR_VMX_UNEXPECTED_EXCEPTION;
11731#endif
11732 }
11733
11734 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11735 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11736
11737 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11738 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11739 AssertRCReturn(rc, rc);
11740
11741 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11742 uint32_t cbOp = 0;
11743 PVM pVM = pVCpu->CTX_SUFF(pVM);
11744 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
11745 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11746 if (RT_SUCCESS(rc))
11747 {
11748 rc = VINF_SUCCESS;
11749 Assert(cbOp == pDis->cbInstr);
11750 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11751 switch (pDis->pCurInstr->uOpcode)
11752 {
11753 case OP_CLI:
11754 {
11755 pMixedCtx->eflags.Bits.u1IF = 0;
11756 pMixedCtx->eflags.Bits.u1RF = 0;
11757 pMixedCtx->rip += pDis->cbInstr;
11758 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11759 if ( !fDbgStepping
11760 && pMixedCtx->eflags.Bits.u1TF)
11761 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
11762 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11763 break;
11764 }
11765
11766 case OP_STI:
11767 {
11768 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11769 pMixedCtx->eflags.Bits.u1IF = 1;
11770 pMixedCtx->eflags.Bits.u1RF = 0;
11771 pMixedCtx->rip += pDis->cbInstr;
11772 if (!fOldIF)
11773 {
11774 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11775 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11776 }
11777 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11778 if ( !fDbgStepping
11779 && pMixedCtx->eflags.Bits.u1TF)
11780 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
11781 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11782 break;
11783 }
11784
11785 case OP_HLT:
11786 {
11787 rc = VINF_EM_HALT;
11788 pMixedCtx->rip += pDis->cbInstr;
11789 pMixedCtx->eflags.Bits.u1RF = 0;
11790 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11791 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11792 break;
11793 }
11794
11795 case OP_POPF:
11796 {
11797 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11798 uint32_t cbParm;
11799 uint32_t uMask;
11800 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11801 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11802 {
11803 cbParm = 4;
11804 uMask = 0xffffffff;
11805 }
11806 else
11807 {
11808 cbParm = 2;
11809 uMask = 0xffff;
11810 }
11811
11812 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11813 RTGCPTR GCPtrStack = 0;
11814 X86EFLAGS Eflags;
11815 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11816 &GCPtrStack);
11817 if (RT_SUCCESS(rc))
11818 {
11819 Assert(sizeof(Eflags.u32) >= cbParm);
11820 Eflags.u32 = 0;
11821 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
11822 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
11823 }
11824 if (RT_FAILURE(rc))
11825 {
11826 rc = VERR_EM_INTERPRETER;
11827 break;
11828 }
11829 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11830 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
11831 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11832 pMixedCtx->esp += cbParm;
11833 pMixedCtx->esp &= uMask;
11834 pMixedCtx->rip += pDis->cbInstr;
11835 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11836 | HM_CHANGED_GUEST_RSP
11837 | HM_CHANGED_GUEST_RFLAGS);
11838 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
11839 POPF restores EFLAGS.TF. */
11840 if ( !fDbgStepping
11841 && fGstStepping)
11842 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
11843 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11844 break;
11845 }
11846
11847 case OP_PUSHF:
11848 {
11849 uint32_t cbParm;
11850 uint32_t uMask;
11851 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11852 {
11853 cbParm = 4;
11854 uMask = 0xffffffff;
11855 }
11856 else
11857 {
11858 cbParm = 2;
11859 uMask = 0xffff;
11860 }
11861
11862 /* Get the stack pointer & push the contents of eflags onto the stack. */
11863 RTGCPTR GCPtrStack = 0;
11864 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11865 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11866 if (RT_FAILURE(rc))
11867 {
11868 rc = VERR_EM_INTERPRETER;
11869 break;
11870 }
11871 X86EFLAGS Eflags = pMixedCtx->eflags;
11872 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11873 Eflags.Bits.u1RF = 0;
11874 Eflags.Bits.u1VM = 0;
11875
11876 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
11877 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11878 {
11879 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
11880 rc = VERR_EM_INTERPRETER;
11881 break;
11882 }
11883 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11884 pMixedCtx->esp -= cbParm;
11885 pMixedCtx->esp &= uMask;
11886 pMixedCtx->rip += pDis->cbInstr;
11887 pMixedCtx->eflags.Bits.u1RF = 0;
11888 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11889 | HM_CHANGED_GUEST_RSP
11890 | HM_CHANGED_GUEST_RFLAGS);
11891 if ( !fDbgStepping
11892 && pMixedCtx->eflags.Bits.u1TF)
11893 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
11894 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11895 break;
11896 }
11897
11898 case OP_IRET:
11899 {
11900 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11901 * instruction reference. */
11902 RTGCPTR GCPtrStack = 0;
11903 uint32_t uMask = 0xffff;
11904 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11905 uint16_t aIretFrame[3];
11906 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11907 {
11908 rc = VERR_EM_INTERPRETER;
11909 break;
11910 }
11911 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11912 &GCPtrStack);
11913 if (RT_SUCCESS(rc))
11914 {
11915 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
11916 PGMACCESSORIGIN_HM));
11917 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
11918 }
11919 if (RT_FAILURE(rc))
11920 {
11921 rc = VERR_EM_INTERPRETER;
11922 break;
11923 }
11924 pMixedCtx->eip = 0;
11925 pMixedCtx->ip = aIretFrame[0];
11926 pMixedCtx->cs.Sel = aIretFrame[1];
11927 pMixedCtx->cs.ValidSel = aIretFrame[1];
11928 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11929 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
11930 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11931 pMixedCtx->sp += sizeof(aIretFrame);
11932 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11933 | HM_CHANGED_GUEST_SEGMENT_REGS
11934 | HM_CHANGED_GUEST_RSP
11935 | HM_CHANGED_GUEST_RFLAGS);
11936 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11937 if ( !fDbgStepping
11938 && fGstStepping)
11939 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
11940 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11941 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11942 break;
11943 }
11944
11945 case OP_INT:
11946 {
11947 uint16_t uVector = pDis->Param1.uValue & 0xff;
11948 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11949 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11950 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11951 break;
11952 }
11953
11954 case OP_INTO:
11955 {
11956 if (pMixedCtx->eflags.Bits.u1OF)
11957 {
11958 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11959 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11960 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11961 }
11962 else
11963 {
11964 pMixedCtx->eflags.Bits.u1RF = 0;
11965 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11966 }
11967 break;
11968 }
11969
11970 default:
11971 {
11972 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
11973 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11974 EMCODETYPE_SUPERVISOR);
11975 rc = VBOXSTRICTRC_VAL(rc2);
11976 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11977 /** @todo We have to set pending-debug exceptions here when the guest is
11978 * single-stepping depending on the instruction that was interpreted. */
11979 Log4(("#GP rc=%Rrc\n", rc));
11980 break;
11981 }
11982 }
11983 }
11984 else
11985 rc = VERR_EM_INTERPRETER;
11986
11987 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11988 ("#GP Unexpected rc=%Rrc\n", rc));
11989 return rc;
11990}
11991
11992
11993#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11994/**
11995 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11996 * the exception reported in the VMX transient structure back into the VM.
11997 *
11998 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11999 * up-to-date.
12000 */
12001static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12002{
12003 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12004
12005 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12006 hmR0VmxCheckExitDueToEventDelivery(). */
12007 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12008 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12009 AssertRCReturn(rc, rc);
12010 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12011
12012#ifdef DEBUG_ramshankar
12013 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12014 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12015 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12016#endif
12017
12018 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12019 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12020 return VINF_SUCCESS;
12021}
12022#endif
12023
12024
12025/**
12026 * VM-exit exception handler for \#PF (Page-fault exception).
12027 */
12028static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12029{
12030 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12031 PVM pVM = pVCpu->CTX_SUFF(pVM);
12032 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12033 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12034 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12035 AssertRCReturn(rc, rc);
12036
12037#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12038 if (pVM->hm.s.fNestedPaging)
12039 {
12040 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12041 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12042 {
12043 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12044 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12045 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12046 }
12047 else
12048 {
12049 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12050 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12051 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12052 }
12053 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12054 return rc;
12055 }
12056#else
12057 Assert(!pVM->hm.s.fNestedPaging);
12058 NOREF(pVM);
12059#endif
12060
12061 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12062 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12063 if (pVmxTransient->fVectoringPF)
12064 {
12065 Assert(pVCpu->hm.s.Event.fPending);
12066 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12067 }
12068
12069 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12070 AssertRCReturn(rc, rc);
12071
12072 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12073 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12074
12075 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12076 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12077 (RTGCPTR)pVmxTransient->uExitQualification);
12078
12079 Log4(("#PF: rc=%Rrc\n", rc));
12080 if (rc == VINF_SUCCESS)
12081 {
12082 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12083 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12084 * memory? We don't update the whole state here... */
12085 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12086 | HM_CHANGED_GUEST_RSP
12087 | HM_CHANGED_GUEST_RFLAGS
12088 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12089 TRPMResetTrap(pVCpu);
12090 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12091 return rc;
12092 }
12093
12094 if (rc == VINF_EM_RAW_GUEST_TRAP)
12095 {
12096 if (!pVmxTransient->fVectoringDoublePF)
12097 {
12098 /* It's a guest page fault and needs to be reflected to the guest. */
12099 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12100 TRPMResetTrap(pVCpu);
12101 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12102 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12103 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12104 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12105 }
12106 else
12107 {
12108 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12109 TRPMResetTrap(pVCpu);
12110 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12111 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12112 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12113 }
12114
12115 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12116 return VINF_SUCCESS;
12117 }
12118
12119 TRPMResetTrap(pVCpu);
12120 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12121 return rc;
12122}
12123
12124/** @} */
12125
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette