VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 57493

Last change on this file since 57493 was 57477, checked in by vboxsync, 9 years ago

VMM/HM: Implement PAUSE filter exiting for VT-x & AMD-V. Also corrected some reserved bits in EPT_VPID capability MSRs.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 505.1 KB
Line 
1/* $Id: HMVMXR0.cpp 57477 2015-08-20 14:20:26Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#include "HMInternal.h"
38#include <VBox/vmm/vm.h>
39#include "HMVMXR0.h"
40#include "dtrace/VBoxVMM.h"
41
42#ifdef DEBUG_ramshankar
43# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
44# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_CHECK_GUEST_STATE
47# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
48# define HMVMX_ALWAYS_TRAP_PF
49# define HMVMX_ALWAYS_SWAP_FPU_STATE
50# define HMVMX_ALWAYS_FLUSH_TLB
51# define HMVMX_ALWAYS_SWAP_EFER
52#endif
53
54
55/*********************************************************************************************************************************
56* Defined Constants And Macros *
57*********************************************************************************************************************************/
58/** Use the function table. */
59#define HMVMX_USE_FUNCTION_TABLE
60
61/** Determine which tagged-TLB flush handler to use. */
62#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
63#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
64#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
65#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
66
67/** @name Updated-guest-state flags.
68 * @{ */
69#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
70#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
71#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
72#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
73#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
74#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
75#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
76#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
77#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
78#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
79#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
80#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
81#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
82#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
83#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
84#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
85#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
86#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
87#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
88#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
89#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
90 | HMVMX_UPDATED_GUEST_RSP \
91 | HMVMX_UPDATED_GUEST_RFLAGS \
92 | HMVMX_UPDATED_GUEST_CR0 \
93 | HMVMX_UPDATED_GUEST_CR3 \
94 | HMVMX_UPDATED_GUEST_CR4 \
95 | HMVMX_UPDATED_GUEST_GDTR \
96 | HMVMX_UPDATED_GUEST_IDTR \
97 | HMVMX_UPDATED_GUEST_LDTR \
98 | HMVMX_UPDATED_GUEST_TR \
99 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
100 | HMVMX_UPDATED_GUEST_DEBUG \
101 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
102 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
104 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
105 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
106 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
107 | HMVMX_UPDATED_GUEST_INTR_STATE \
108 | HMVMX_UPDATED_GUEST_APIC_STATE)
109/** @} */
110
111/** @name
112 * Flags to skip redundant reads of some common VMCS fields that are not part of
113 * the guest-CPU state but are in the transient structure.
114 */
115#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
117#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
122/** @} */
123
124/** @name
125 * States of the VMCS.
126 *
127 * This does not reflect all possible VMCS states but currently only those
128 * needed for maintaining the VMCS consistently even when thread-context hooks
129 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
130 */
131#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
132#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
133#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
134/** @} */
135
136/**
137 * Exception bitmap mask for real-mode guests (real-on-v86).
138 *
139 * We need to intercept all exceptions manually except:
140 * - #NM, #MF handled in hmR0VmxLoadSharedCR0().
141 * - #DB handled in hmR0VmxLoadSharedDebugState().
142 * - #PF need not be intercepted even in real-mode if we have Nested Paging
143 * support.
144 */
145#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
146 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
147 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
148 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
149 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
150 /* RT_BIT(X86_XCPT_MF) */ | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
151 | RT_BIT(X86_XCPT_XF))
152
153/**
154 * Exception bitmap mask for all contributory exceptions.
155 *
156 * Page fault is deliberately excluded here as it's conditional as to whether
157 * it's contributory or benign. Page faults are handled separately.
158 */
159#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
160 | RT_BIT(X86_XCPT_DE))
161
162/** Maximum VM-instruction error number. */
163#define HMVMX_INSTR_ERROR_MAX 28
164
165/** Profiling macro. */
166#ifdef HM_PROFILE_EXIT_DISPATCH
167# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
168# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
169#else
170# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
171# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
172#endif
173
174/** Assert that preemption is disabled or covered by thread-context hooks. */
175#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
176 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
177
178/** Assert that we haven't migrated CPUs when thread-context hooks are not
179 * used. */
180#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
181 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
182 ("Illegal migration! Entered on CPU %u Current %u\n", \
183 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
184
185/** Helper macro for VM-exit handlers called unexpectedly. */
186#define HMVMX_RETURN_UNEXPECTED_EXIT() \
187 do { \
188 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
189 return VERR_VMX_UNEXPECTED_EXIT; \
190 } while (0)
191
192
193/*********************************************************************************************************************************
194* Structures and Typedefs *
195*********************************************************************************************************************************/
196/**
197 * VMX transient state.
198 *
199 * A state structure for holding miscellaneous information across
200 * VMX non-root operation and restored after the transition.
201 */
202typedef struct VMXTRANSIENT
203{
204 /** The host's rflags/eflags. */
205 RTCCUINTREG fEFlags;
206#if HC_ARCH_BITS == 32
207 uint32_t u32Alignment0;
208#endif
209 /** The guest's TPR value used for TPR shadowing. */
210 uint8_t u8GuestTpr;
211 /** Alignment. */
212 uint8_t abAlignment0[7];
213
214 /** The basic VM-exit reason. */
215 uint16_t uExitReason;
216 /** Alignment. */
217 uint16_t u16Alignment0;
218 /** The VM-exit interruption error code. */
219 uint32_t uExitIntErrorCode;
220 /** The VM-exit exit code qualification. */
221 uint64_t uExitQualification;
222
223 /** The VM-exit interruption-information field. */
224 uint32_t uExitIntInfo;
225 /** The VM-exit instruction-length field. */
226 uint32_t cbInstr;
227 /** The VM-exit instruction-information field. */
228 union
229 {
230 /** Plain unsigned int representation. */
231 uint32_t u;
232 /** INS and OUTS information. */
233 struct
234 {
235 uint32_t u6Reserved0 : 7;
236 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
237 uint32_t u3AddrSize : 3;
238 uint32_t u5Reserved1 : 5;
239 /** The segment register (X86_SREG_XXX). */
240 uint32_t iSegReg : 3;
241 uint32_t uReserved2 : 14;
242 } StrIo;
243 } ExitInstrInfo;
244 /** Whether the VM-entry failed or not. */
245 bool fVMEntryFailed;
246 /** Alignment. */
247 uint8_t abAlignment1[3];
248
249 /** The VM-entry interruption-information field. */
250 uint32_t uEntryIntInfo;
251 /** The VM-entry exception error code field. */
252 uint32_t uEntryXcptErrorCode;
253 /** The VM-entry instruction length field. */
254 uint32_t cbEntryInstr;
255
256 /** IDT-vectoring information field. */
257 uint32_t uIdtVectoringInfo;
258 /** IDT-vectoring error code. */
259 uint32_t uIdtVectoringErrorCode;
260
261 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
262 uint32_t fVmcsFieldsRead;
263
264 /** Whether the guest FPU was active at the time of VM-exit. */
265 bool fWasGuestFPUStateActive;
266 /** Whether the guest debug state was active at the time of VM-exit. */
267 bool fWasGuestDebugStateActive;
268 /** Whether the hyper debug state was active at the time of VM-exit. */
269 bool fWasHyperDebugStateActive;
270 /** Whether TSC-offsetting should be setup before VM-entry. */
271 bool fUpdateTscOffsettingAndPreemptTimer;
272 /** Whether the VM-exit was caused by a page-fault during delivery of a
273 * contributory exception or a page-fault. */
274 bool fVectoringDoublePF;
275 /** Whether the VM-exit was caused by a page-fault during delivery of an
276 * external interrupt or NMI. */
277 bool fVectoringPF;
278} VMXTRANSIENT;
279AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284/** Pointer to VMX transient state. */
285typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288/**
289 * MSR-bitmap read permissions.
290 */
291typedef enum VMXMSREXITREAD
292{
293 /** Reading this MSR causes a VM-exit. */
294 VMXMSREXIT_INTERCEPT_READ = 0xb,
295 /** Reading this MSR does not cause a VM-exit. */
296 VMXMSREXIT_PASSTHRU_READ
297} VMXMSREXITREAD;
298/** Pointer to MSR-bitmap read permissions. */
299typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301/**
302 * MSR-bitmap write permissions.
303 */
304typedef enum VMXMSREXITWRITE
305{
306 /** Writing to this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308 /** Writing to this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_WRITE
310} VMXMSREXITWRITE;
311/** Pointer to MSR-bitmap write permissions. */
312typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315/**
316 * VMX VM-exit handler.
317 *
318 * @returns VBox status code.
319 * @param pVCpu Pointer to the VMCPU.
320 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
321 * out-of-sync. Make sure to update the required
322 * fields before using them.
323 * @param pVmxTransient Pointer to the VMX-transient structure.
324 */
325#ifndef HMVMX_USE_FUNCTION_TABLE
326typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327#else
328typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329/** Pointer to VM-exit handler. */
330typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331#endif
332
333
334/*********************************************************************************************************************************
335* Internal Functions *
336*********************************************************************************************************************************/
337static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
338static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
339static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
341 bool fStepping, uint32_t *puIntState);
342#if HC_ARCH_BITS == 32
343static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
344#endif
345#ifndef HMVMX_USE_FUNCTION_TABLE
346DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
347# define HMVMX_EXIT_DECL static int
348#else
349# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
350#endif
351DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
352 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
353
354/** @name VM-exit handlers.
355 * @{
356 */
357static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
358static FNVMXEXITHANDLER hmR0VmxExitExtInt;
359static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
360static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
361static FNVMXEXITHANDLER hmR0VmxExitSipi;
362static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
363static FNVMXEXITHANDLER hmR0VmxExitSmi;
364static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
365static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
366static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
367static FNVMXEXITHANDLER hmR0VmxExitCpuid;
368static FNVMXEXITHANDLER hmR0VmxExitGetsec;
369static FNVMXEXITHANDLER hmR0VmxExitHlt;
370static FNVMXEXITHANDLER hmR0VmxExitInvd;
371static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
372static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
373static FNVMXEXITHANDLER hmR0VmxExitVmcall;
374static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
375static FNVMXEXITHANDLER hmR0VmxExitRsm;
376static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
377static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
378static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
379static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
380static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
381static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
382static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
383static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
384static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
385static FNVMXEXITHANDLER hmR0VmxExitMwait;
386static FNVMXEXITHANDLER hmR0VmxExitMtf;
387static FNVMXEXITHANDLER hmR0VmxExitMonitor;
388static FNVMXEXITHANDLER hmR0VmxExitPause;
389static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
390static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
391static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
392static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
393static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
394static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
395static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
396static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
397static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
398static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
399static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
400static FNVMXEXITHANDLER hmR0VmxExitRdrand;
401static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
402/** @} */
403
404static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
411static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
412#endif
413static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
414
415
416/*********************************************************************************************************************************
417* Global Variables *
418*********************************************************************************************************************************/
419#ifdef HMVMX_USE_FUNCTION_TABLE
420
421/**
422 * VMX_EXIT dispatch table.
423 */
424static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
425{
426 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
427 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
428 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
429 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
430 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
431 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
432 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
433 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
434 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
435 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
436 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
437 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
438 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
439 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
440 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
441 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
442 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
443 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
444 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
445 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
446 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
447 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
448 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
449 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
450 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
451 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
452 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
453 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
454 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
455 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
456 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
457 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
458 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
459 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
460 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
461 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
462 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
463 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
464 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
465 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
466 /* 40 UNDEFINED */ hmR0VmxExitPause,
467 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
468 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
469 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
470 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
471 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
472 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
473 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
474 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
475 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
476 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
477 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
478 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
479 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
480 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
481 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
482 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
483 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
484 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
485 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
486 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
487 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
488 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
489 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
490 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
491};
492#endif /* HMVMX_USE_FUNCTION_TABLE */
493
494#ifdef VBOX_STRICT
495static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
496{
497 /* 0 */ "(Not Used)",
498 /* 1 */ "VMCALL executed in VMX root operation.",
499 /* 2 */ "VMCLEAR with invalid physical address.",
500 /* 3 */ "VMCLEAR with VMXON pointer.",
501 /* 4 */ "VMLAUNCH with non-clear VMCS.",
502 /* 5 */ "VMRESUME with non-launched VMCS.",
503 /* 6 */ "VMRESUME after VMXOFF",
504 /* 7 */ "VM-entry with invalid control fields.",
505 /* 8 */ "VM-entry with invalid host state fields.",
506 /* 9 */ "VMPTRLD with invalid physical address.",
507 /* 10 */ "VMPTRLD with VMXON pointer.",
508 /* 11 */ "VMPTRLD with incorrect revision identifier.",
509 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
510 /* 13 */ "VMWRITE to read-only VMCS component.",
511 /* 14 */ "(Not Used)",
512 /* 15 */ "VMXON executed in VMX root operation.",
513 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
514 /* 17 */ "VM-entry with non-launched executing VMCS.",
515 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
516 /* 19 */ "VMCALL with non-clear VMCS.",
517 /* 20 */ "VMCALL with invalid VM-exit control fields.",
518 /* 21 */ "(Not Used)",
519 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
520 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
521 /* 24 */ "VMCALL with invalid SMM-monitor features.",
522 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
523 /* 26 */ "VM-entry with events blocked by MOV SS.",
524 /* 27 */ "(Not Used)",
525 /* 28 */ "Invalid operand to INVEPT/INVVPID."
526};
527#endif /* VBOX_STRICT */
528
529
530
531/**
532 * Updates the VM's last error record. If there was a VMX instruction error,
533 * reads the error data from the VMCS and updates VCPU's last error record as
534 * well.
535 *
536 * @param pVM Pointer to the VM.
537 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
538 * VERR_VMX_UNABLE_TO_START_VM or
539 * VERR_VMX_INVALID_VMCS_FIELD).
540 * @param rc The error code.
541 */
542static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
543{
544 AssertPtr(pVM);
545 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
546 || rc == VERR_VMX_UNABLE_TO_START_VM)
547 {
548 AssertPtrReturnVoid(pVCpu);
549 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
550 }
551 pVM->hm.s.lLastError = rc;
552}
553
554
555/**
556 * Reads the VM-entry interruption-information field from the VMCS into the VMX
557 * transient structure.
558 *
559 * @returns VBox status code.
560 * @param pVmxTransient Pointer to the VMX transient structure.
561 *
562 * @remarks No-long-jump zone!!!
563 */
564DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
565{
566 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
567 AssertRCReturn(rc, rc);
568 return VINF_SUCCESS;
569}
570
571
572/**
573 * Reads the VM-entry exception error code field from the VMCS into
574 * the VMX transient structure.
575 *
576 * @returns VBox status code.
577 * @param pVmxTransient Pointer to the VMX transient structure.
578 *
579 * @remarks No-long-jump zone!!!
580 */
581DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
582{
583 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
584 AssertRCReturn(rc, rc);
585 return VINF_SUCCESS;
586}
587
588
589/**
590 * Reads the VM-entry exception error code field from the VMCS into
591 * the VMX transient structure.
592 *
593 * @returns VBox status code.
594 * @param pVmxTransient Pointer to the VMX transient structure.
595 *
596 * @remarks No-long-jump zone!!!
597 */
598DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
599{
600 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
601 AssertRCReturn(rc, rc);
602 return VINF_SUCCESS;
603}
604
605
606/**
607 * Reads the VM-exit interruption-information field from the VMCS into the VMX
608 * transient structure.
609 *
610 * @returns VBox status code.
611 * @param pVmxTransient Pointer to the VMX transient structure.
612 */
613DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
614{
615 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
616 {
617 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
618 AssertRCReturn(rc, rc);
619 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
620 }
621 return VINF_SUCCESS;
622}
623
624
625/**
626 * Reads the VM-exit interruption error code from the VMCS into the VMX
627 * transient structure.
628 *
629 * @returns VBox status code.
630 * @param pVmxTransient Pointer to the VMX transient structure.
631 */
632DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
633{
634 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
635 {
636 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
637 AssertRCReturn(rc, rc);
638 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
639 }
640 return VINF_SUCCESS;
641}
642
643
644/**
645 * Reads the VM-exit instruction length field from the VMCS into the VMX
646 * transient structure.
647 *
648 * @returns VBox status code.
649 * @param pVCpu Pointer to the VMCPU.
650 * @param pVmxTransient Pointer to the VMX transient structure.
651 */
652DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
653{
654 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
655 {
656 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
657 AssertRCReturn(rc, rc);
658 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
659 }
660 return VINF_SUCCESS;
661}
662
663
664/**
665 * Reads the VM-exit instruction-information field from the VMCS into
666 * the VMX transient structure.
667 *
668 * @returns VBox status code.
669 * @param pVmxTransient Pointer to the VMX transient structure.
670 */
671DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
672{
673 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
674 {
675 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
676 AssertRCReturn(rc, rc);
677 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
678 }
679 return VINF_SUCCESS;
680}
681
682
683/**
684 * Reads the exit code qualification from the VMCS into the VMX transient
685 * structure.
686 *
687 * @returns VBox status code.
688 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
689 * case).
690 * @param pVmxTransient Pointer to the VMX transient structure.
691 */
692DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
693{
694 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
695 {
696 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
697 AssertRCReturn(rc, rc);
698 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
699 }
700 return VINF_SUCCESS;
701}
702
703
704/**
705 * Reads the IDT-vectoring information field from the VMCS into the VMX
706 * transient structure.
707 *
708 * @returns VBox status code.
709 * @param pVmxTransient Pointer to the VMX transient structure.
710 *
711 * @remarks No-long-jump zone!!!
712 */
713DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
714{
715 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
716 {
717 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
718 AssertRCReturn(rc, rc);
719 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
720 }
721 return VINF_SUCCESS;
722}
723
724
725/**
726 * Reads the IDT-vectoring error code from the VMCS into the VMX
727 * transient structure.
728 *
729 * @returns VBox status code.
730 * @param pVmxTransient Pointer to the VMX transient structure.
731 */
732DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
733{
734 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
735 {
736 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
737 AssertRCReturn(rc, rc);
738 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
739 }
740 return VINF_SUCCESS;
741}
742
743
744/**
745 * Enters VMX root mode operation on the current CPU.
746 *
747 * @returns VBox status code.
748 * @param pVM Pointer to the VM (optional, can be NULL, after
749 * a resume).
750 * @param HCPhysCpuPage Physical address of the VMXON region.
751 * @param pvCpuPage Pointer to the VMXON region.
752 */
753static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
754{
755 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
756 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
757 Assert(pvCpuPage);
758 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
759
760 if (pVM)
761 {
762 /* Write the VMCS revision dword to the VMXON region. */
763 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
764 }
765
766 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
767 RTCCUINTREG fEFlags = ASMIntDisableFlags();
768
769 /* Enable the VMX bit in CR4 if necessary. */
770 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
771
772 /* Enter VMX root mode. */
773 int rc = VMXEnable(HCPhysCpuPage);
774 if (RT_FAILURE(rc))
775 {
776 if (!(uOldCr4 & X86_CR4_VMXE))
777 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
778
779 if (pVM)
780 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
781 }
782
783 /* Restore interrupts. */
784 ASMSetFlags(fEFlags);
785 return rc;
786}
787
788
789/**
790 * Exits VMX root mode operation on the current CPU.
791 *
792 * @returns VBox status code.
793 */
794static int hmR0VmxLeaveRootMode(void)
795{
796 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
797
798 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
799 RTCCUINTREG fEFlags = ASMIntDisableFlags();
800
801 /* If we're for some reason not in VMX root mode, then don't leave it. */
802 RTCCUINTREG uHostCR4 = ASMGetCR4();
803
804 int rc;
805 if (uHostCR4 & X86_CR4_VMXE)
806 {
807 /* Exit VMX root mode and clear the VMX bit in CR4. */
808 VMXDisable();
809 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
810 rc = VINF_SUCCESS;
811 }
812 else
813 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
814
815 /* Restore interrupts. */
816 ASMSetFlags(fEFlags);
817 return rc;
818}
819
820
821/**
822 * Allocates and maps one physically contiguous page. The allocated page is
823 * zero'd out. (Used by various VT-x structures).
824 *
825 * @returns IPRT status code.
826 * @param pMemObj Pointer to the ring-0 memory object.
827 * @param ppVirt Where to store the virtual address of the
828 * allocation.
829 * @param pPhys Where to store the physical address of the
830 * allocation.
831 */
832DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
833{
834 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
835 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
836 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
837
838 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
839 if (RT_FAILURE(rc))
840 return rc;
841 *ppVirt = RTR0MemObjAddress(*pMemObj);
842 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
843 ASMMemZero32(*ppVirt, PAGE_SIZE);
844 return VINF_SUCCESS;
845}
846
847
848/**
849 * Frees and unmaps an allocated physical page.
850 *
851 * @param pMemObj Pointer to the ring-0 memory object.
852 * @param ppVirt Where to re-initialize the virtual address of
853 * allocation as 0.
854 * @param pHCPhys Where to re-initialize the physical address of the
855 * allocation as 0.
856 */
857DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
858{
859 AssertPtr(pMemObj);
860 AssertPtr(ppVirt);
861 AssertPtr(pHCPhys);
862 if (*pMemObj != NIL_RTR0MEMOBJ)
863 {
864 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
865 AssertRC(rc);
866 *pMemObj = NIL_RTR0MEMOBJ;
867 *ppVirt = 0;
868 *pHCPhys = 0;
869 }
870}
871
872
873/**
874 * Worker function to free VT-x related structures.
875 *
876 * @returns IPRT status code.
877 * @param pVM Pointer to the VM.
878 */
879static void hmR0VmxStructsFree(PVM pVM)
880{
881 for (VMCPUID i = 0; i < pVM->cCpus; i++)
882 {
883 PVMCPU pVCpu = &pVM->aCpus[i];
884 AssertPtr(pVCpu);
885
886 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
887 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
888
889 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
890 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
891
892 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
893 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
894 }
895
896 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
897#ifdef VBOX_WITH_CRASHDUMP_MAGIC
898 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
899#endif
900}
901
902
903/**
904 * Worker function to allocate VT-x related VM structures.
905 *
906 * @returns IPRT status code.
907 * @param pVM Pointer to the VM.
908 */
909static int hmR0VmxStructsAlloc(PVM pVM)
910{
911 /*
912 * Initialize members up-front so we can cleanup properly on allocation failure.
913 */
914#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
915 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
916 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
917 pVM->hm.s.vmx.HCPhys##a_Name = 0;
918
919#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
920 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
921 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
922 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
923
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
926#endif
927 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
928
929 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
930 for (VMCPUID i = 0; i < pVM->cCpus; i++)
931 {
932 PVMCPU pVCpu = &pVM->aCpus[i];
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
935 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
936 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
937 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
938 }
939#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
940#undef VMXLOCAL_INIT_VM_MEMOBJ
941
942 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
943 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
944 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
945 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
946
947 /*
948 * Allocate all the VT-x structures.
949 */
950 int rc = VINF_SUCCESS;
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
953 if (RT_FAILURE(rc))
954 goto cleanup;
955 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
956 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
957#endif
958
959 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
960 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
961 {
962 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
963 &pVM->hm.s.vmx.HCPhysApicAccess);
964 if (RT_FAILURE(rc))
965 goto cleanup;
966 }
967
968 /*
969 * Initialize per-VCPU VT-x structures.
970 */
971 for (VMCPUID i = 0; i < pVM->cCpus; i++)
972 {
973 PVMCPU pVCpu = &pVM->aCpus[i];
974 AssertPtr(pVCpu);
975
976 /* Allocate the VM control structure (VMCS). */
977 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
978 if (RT_FAILURE(rc))
979 goto cleanup;
980
981 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
982 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
983 {
984 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
985 &pVCpu->hm.s.vmx.HCPhysVirtApic);
986 if (RT_FAILURE(rc))
987 goto cleanup;
988 }
989
990 /*
991 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
992 * transparent accesses of specific MSRs.
993 *
994 * If the condition for enabling MSR bitmaps changes here, don't forget to
995 * update HMAreMsrBitmapsAvailable().
996 */
997 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
998 {
999 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1000 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1001 if (RT_FAILURE(rc))
1002 goto cleanup;
1003 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1004 }
1005
1006 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1007 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1008 if (RT_FAILURE(rc))
1009 goto cleanup;
1010
1011 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1012 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 return VINF_SUCCESS;
1018
1019cleanup:
1020 hmR0VmxStructsFree(pVM);
1021 return rc;
1022}
1023
1024
1025/**
1026 * Does global VT-x initialization (called during module initialization).
1027 *
1028 * @returns VBox status code.
1029 */
1030VMMR0DECL(int) VMXR0GlobalInit(void)
1031{
1032#ifdef HMVMX_USE_FUNCTION_TABLE
1033 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1034# ifdef VBOX_STRICT
1035 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1036 Assert(g_apfnVMExitHandlers[i]);
1037# endif
1038#endif
1039 return VINF_SUCCESS;
1040}
1041
1042
1043/**
1044 * Does global VT-x termination (called during module termination).
1045 */
1046VMMR0DECL(void) VMXR0GlobalTerm()
1047{
1048 /* Nothing to do currently. */
1049}
1050
1051
1052/**
1053 * Sets up and activates VT-x on the current CPU.
1054 *
1055 * @returns VBox status code.
1056 * @param pCpu Pointer to the global CPU info struct.
1057 * @param pVM Pointer to the VM (can be NULL after a host resume
1058 * operation).
1059 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1060 * fEnabledByHost is true).
1061 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1062 * @a fEnabledByHost is true).
1063 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1064 * enable VT-x on the host.
1065 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1066 */
1067VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1068 void *pvMsrs)
1069{
1070 Assert(pCpu);
1071 Assert(pvMsrs);
1072 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1073
1074 /* Enable VT-x if it's not already enabled by the host. */
1075 if (!fEnabledByHost)
1076 {
1077 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1078 if (RT_FAILURE(rc))
1079 return rc;
1080 }
1081
1082 /*
1083 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1084 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1085 */
1086 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1087 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1088 {
1089 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1090 pCpu->fFlushAsidBeforeUse = false;
1091 }
1092 else
1093 pCpu->fFlushAsidBeforeUse = true;
1094
1095 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1096 ++pCpu->cTlbFlushes;
1097
1098 return VINF_SUCCESS;
1099}
1100
1101
1102/**
1103 * Deactivates VT-x on the current CPU.
1104 *
1105 * @returns VBox status code.
1106 * @param pCpu Pointer to the global CPU info struct.
1107 * @param pvCpuPage Pointer to the VMXON region.
1108 * @param HCPhysCpuPage Physical address of the VMXON region.
1109 *
1110 * @remarks This function should never be called when SUPR0EnableVTx() or
1111 * similar was used to enable VT-x on the host.
1112 */
1113VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1114{
1115 NOREF(pCpu);
1116 NOREF(pvCpuPage);
1117 NOREF(HCPhysCpuPage);
1118
1119 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1120 return hmR0VmxLeaveRootMode();
1121}
1122
1123
1124/**
1125 * Sets the permission bits for the specified MSR in the MSR bitmap.
1126 *
1127 * @param pVCpu Pointer to the VMCPU.
1128 * @param uMSR The MSR value.
1129 * @param enmRead Whether reading this MSR causes a VM-exit.
1130 * @param enmWrite Whether writing this MSR causes a VM-exit.
1131 */
1132static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1133{
1134 int32_t iBit;
1135 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1136
1137 /*
1138 * Layout:
1139 * 0x000 - 0x3ff - Low MSR read bits
1140 * 0x400 - 0x7ff - High MSR read bits
1141 * 0x800 - 0xbff - Low MSR write bits
1142 * 0xc00 - 0xfff - High MSR write bits
1143 */
1144 if (uMsr <= 0x00001FFF)
1145 iBit = uMsr;
1146 else if ( uMsr >= 0xC0000000
1147 && uMsr <= 0xC0001FFF)
1148 {
1149 iBit = (uMsr - 0xC0000000);
1150 pbMsrBitmap += 0x400;
1151 }
1152 else
1153 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1154
1155 Assert(iBit <= 0x1fff);
1156 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1157 ASMBitSet(pbMsrBitmap, iBit);
1158 else
1159 ASMBitClear(pbMsrBitmap, iBit);
1160
1161 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1162 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1163 else
1164 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1165}
1166
1167
1168#ifdef VBOX_STRICT
1169/**
1170 * Gets the permission bits for the specified MSR in the MSR bitmap.
1171 *
1172 * @returns VBox status code.
1173 * @retval VINF_SUCCESS if the specified MSR is found.
1174 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1175 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1176 *
1177 * @param pVCpu Pointer to the VMCPU.
1178 * @param uMsr The MSR.
1179 * @param penmRead Where to store the read permissions.
1180 * @param penmWrite Where to store the write permissions.
1181 */
1182static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1183{
1184 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1185 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1186 int32_t iBit;
1187 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1188
1189 /* See hmR0VmxSetMsrPermission() for the layout. */
1190 if (uMsr <= 0x00001FFF)
1191 iBit = uMsr;
1192 else if ( uMsr >= 0xC0000000
1193 && uMsr <= 0xC0001FFF)
1194 {
1195 iBit = (uMsr - 0xC0000000);
1196 pbMsrBitmap += 0x400;
1197 }
1198 else
1199 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1200
1201 Assert(iBit <= 0x1fff);
1202 if (ASMBitTest(pbMsrBitmap, iBit))
1203 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1204 else
1205 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1206
1207 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1208 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1209 else
1210 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1211 return VINF_SUCCESS;
1212}
1213#endif /* VBOX_STRICT */
1214
1215
1216/**
1217 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1218 * area.
1219 *
1220 * @returns VBox status code.
1221 * @param pVCpu Pointer to the VMCPU.
1222 * @param cMsrs The number of MSRs.
1223 */
1224DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1225{
1226 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1227 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1228 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1229 {
1230 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1231 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1232 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1233 }
1234
1235 /* Update number of guest MSRs to load/store across the world-switch. */
1236 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1237 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1238
1239 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1240 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1241
1242 /* Update the VCPU's copy of the MSR count. */
1243 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1244
1245 return VINF_SUCCESS;
1246}
1247
1248
1249/**
1250 * Adds a new (or updates the value of an existing) guest/host MSR
1251 * pair to be swapped during the world-switch as part of the
1252 * auto-load/store MSR area in the VMCS.
1253 *
1254 * @returns VBox status code.
1255 * @param pVCpu Pointer to the VMCPU.
1256 * @param uMsr The MSR.
1257 * @param uGuestMsr Value of the guest MSR.
1258 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1259 * necessary.
1260 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1261 * its value was updated. Optional, can be NULL.
1262 */
1263static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1264 bool *pfAddedAndUpdated)
1265{
1266 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1267 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1268 uint32_t i;
1269 for (i = 0; i < cMsrs; i++)
1270 {
1271 if (pGuestMsr->u32Msr == uMsr)
1272 break;
1273 pGuestMsr++;
1274 }
1275
1276 bool fAdded = false;
1277 if (i == cMsrs)
1278 {
1279 ++cMsrs;
1280 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1281 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1282
1283 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1284 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1285 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1286
1287 fAdded = true;
1288 }
1289
1290 /* Update the MSR values in the auto-load/store MSR area. */
1291 pGuestMsr->u32Msr = uMsr;
1292 pGuestMsr->u64Value = uGuestMsrValue;
1293
1294 /* Create/update the MSR slot in the host MSR area. */
1295 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1296 pHostMsr += i;
1297 pHostMsr->u32Msr = uMsr;
1298
1299 /*
1300 * Update the host MSR only when requested by the caller AND when we're
1301 * adding it to the auto-load/store area. Otherwise, it would have been
1302 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1303 */
1304 bool fUpdatedMsrValue = false;
1305 if ( fAdded
1306 && fUpdateHostMsr)
1307 {
1308 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1309 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1310 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1311 fUpdatedMsrValue = true;
1312 }
1313
1314 if (pfAddedAndUpdated)
1315 *pfAddedAndUpdated = fUpdatedMsrValue;
1316 return VINF_SUCCESS;
1317}
1318
1319
1320/**
1321 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1322 * auto-load/store MSR area in the VMCS.
1323 *
1324 * @returns VBox status code.
1325 * @param pVCpu Pointer to the VMCPU.
1326 * @param uMsr The MSR.
1327 */
1328static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1329{
1330 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1331 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1332 for (uint32_t i = 0; i < cMsrs; i++)
1333 {
1334 /* Find the MSR. */
1335 if (pGuestMsr->u32Msr == uMsr)
1336 {
1337 /* If it's the last MSR, simply reduce the count. */
1338 if (i == cMsrs - 1)
1339 {
1340 --cMsrs;
1341 break;
1342 }
1343
1344 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1345 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1346 pLastGuestMsr += cMsrs - 1;
1347 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1348 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1349
1350 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1351 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1352 pLastHostMsr += cMsrs - 1;
1353 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1354 pHostMsr->u64Value = pLastHostMsr->u64Value;
1355 --cMsrs;
1356 break;
1357 }
1358 pGuestMsr++;
1359 }
1360
1361 /* Update the VMCS if the count changed (meaning the MSR was found). */
1362 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1363 {
1364 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1365 AssertRCReturn(rc, rc);
1366
1367 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1368 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1369 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1370
1371 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1372 return VINF_SUCCESS;
1373 }
1374
1375 return VERR_NOT_FOUND;
1376}
1377
1378
1379/**
1380 * Checks if the specified guest MSR is part of the auto-load/store area in
1381 * the VMCS.
1382 *
1383 * @returns true if found, false otherwise.
1384 * @param pVCpu Pointer to the VMCPU.
1385 * @param uMsr The MSR to find.
1386 */
1387static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1388{
1389 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1390 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1391
1392 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1393 {
1394 if (pGuestMsr->u32Msr == uMsr)
1395 return true;
1396 }
1397 return false;
1398}
1399
1400
1401/**
1402 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1403 *
1404 * @param pVCpu Pointer to the VMCPU.
1405 *
1406 * @remarks No-long-jump zone!!!
1407 */
1408static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1409{
1410 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1411 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1412 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1413 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1414
1415 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1416 {
1417 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1418
1419 /*
1420 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1421 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1422 */
1423 if (pHostMsr->u32Msr == MSR_K6_EFER)
1424 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1425 else
1426 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1427 }
1428
1429 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1430}
1431
1432
1433#if HC_ARCH_BITS == 64
1434/**
1435 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1436 * perform lazy restoration of the host MSRs while leaving VT-x.
1437 *
1438 * @param pVCpu Pointer to the VMCPU.
1439 *
1440 * @remarks No-long-jump zone!!!
1441 */
1442static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1443{
1444 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1445
1446 /*
1447 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1448 */
1449 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1450 {
1451 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1452 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1453 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1454 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1455 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1456 }
1457}
1458
1459
1460/**
1461 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1462 * lazily while leaving VT-x.
1463 *
1464 * @returns true if it does, false otherwise.
1465 * @param pVCpu Pointer to the VMCPU.
1466 * @param uMsr The MSR to check.
1467 */
1468static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1469{
1470 NOREF(pVCpu);
1471 switch (uMsr)
1472 {
1473 case MSR_K8_LSTAR:
1474 case MSR_K6_STAR:
1475 case MSR_K8_SF_MASK:
1476 case MSR_K8_KERNEL_GS_BASE:
1477 return true;
1478 }
1479 return false;
1480}
1481
1482
1483/**
1484 * Saves a set of guest MSRs back into the guest-CPU context.
1485 *
1486 * @param pVCpu Pointer to the VMCPU.
1487 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1488 * out-of-sync. Make sure to update the required fields
1489 * before using them.
1490 *
1491 * @remarks No-long-jump zone!!!
1492 */
1493static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1494{
1495 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1496 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1497
1498 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1499 {
1500 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1501 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1502 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1503 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1504 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1505 }
1506}
1507
1508
1509/**
1510 * Loads a set of guests MSRs to allow read/passthru to the guest.
1511 *
1512 * The name of this function is slightly confusing. This function does NOT
1513 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1514 * common prefix for functions dealing with "lazy restoration" of the shared
1515 * MSRs.
1516 *
1517 * @param pVCpu Pointer to the VMCPU.
1518 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1519 * out-of-sync. Make sure to update the required fields
1520 * before using them.
1521 *
1522 * @remarks No-long-jump zone!!!
1523 */
1524static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1525{
1526 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1527 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1528
1529#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1530 do { \
1531 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1532 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1533 else \
1534 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1535 } while (0)
1536
1537 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1538 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1539 {
1540 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1541 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1542 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1543 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1544 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1545 }
1546 else
1547 {
1548 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1549 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1550 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1551 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1552 }
1553
1554#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1555}
1556
1557
1558/**
1559 * Performs lazy restoration of the set of host MSRs if they were previously
1560 * loaded with guest MSR values.
1561 *
1562 * @param pVCpu Pointer to the VMCPU.
1563 *
1564 * @remarks No-long-jump zone!!!
1565 * @remarks The guest MSRs should have been saved back into the guest-CPU
1566 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1567 */
1568static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1569{
1570 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1571 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1572
1573 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1574 {
1575 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1576 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1577 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1578 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1579 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1580 }
1581 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1582}
1583#endif /* HC_ARCH_BITS == 64 */
1584
1585
1586/**
1587 * Verifies that our cached values of the VMCS controls are all
1588 * consistent with what's actually present in the VMCS.
1589 *
1590 * @returns VBox status code.
1591 * @param pVCpu Pointer to the VMCPU.
1592 */
1593static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1594{
1595 uint32_t u32Val;
1596 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1597 AssertRCReturn(rc, rc);
1598 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1599 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1600
1601 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1602 AssertRCReturn(rc, rc);
1603 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1604 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1605
1606 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1607 AssertRCReturn(rc, rc);
1608 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1609 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1610
1611 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1612 AssertRCReturn(rc, rc);
1613 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1614 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1615
1616 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1617 {
1618 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1619 AssertRCReturn(rc, rc);
1620 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1621 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1622 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1623 }
1624
1625 return VINF_SUCCESS;
1626}
1627
1628
1629#ifdef VBOX_STRICT
1630/**
1631 * Verifies that our cached host EFER value has not changed
1632 * since we cached it.
1633 *
1634 * @param pVCpu Pointer to the VMCPU.
1635 */
1636static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1637{
1638 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1639
1640 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1641 {
1642 uint64_t u64Val;
1643 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1644 AssertRC(rc);
1645
1646 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1647 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1648 }
1649}
1650
1651
1652/**
1653 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1654 * VMCS are correct.
1655 *
1656 * @param pVCpu Pointer to the VMCPU.
1657 */
1658static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1659{
1660 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1661
1662 /* Verify MSR counts in the VMCS are what we think it should be. */
1663 uint32_t cMsrs;
1664 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1665 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1666
1667 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1668 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1669
1670 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1671 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1672
1673 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1674 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1675 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1676 {
1677 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1678 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1679 pGuestMsr->u32Msr, cMsrs));
1680
1681 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1682 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1683 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1684
1685 /* Verify that the permissions are as expected in the MSR bitmap. */
1686 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1687 {
1688 VMXMSREXITREAD enmRead;
1689 VMXMSREXITWRITE enmWrite;
1690 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1691 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1692 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1693 {
1694 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1695 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1696 }
1697 else
1698 {
1699 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1700 pGuestMsr->u32Msr, cMsrs));
1701 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1702 pGuestMsr->u32Msr, cMsrs));
1703 }
1704 }
1705 }
1706}
1707#endif /* VBOX_STRICT */
1708
1709
1710/**
1711 * Flushes the TLB using EPT.
1712 *
1713 * @returns VBox status code.
1714 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1715 * enmFlush).
1716 * @param enmFlush Type of flush.
1717 *
1718 * @remarks Caller is responsible for making sure this function is called only
1719 * when NestedPaging is supported and providing @a enmFlush that is
1720 * supported by the CPU.
1721 * @remarks Can be called with interrupts disabled.
1722 */
1723static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1724{
1725 uint64_t au64Descriptor[2];
1726 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1727 au64Descriptor[0] = 0;
1728 else
1729 {
1730 Assert(pVCpu);
1731 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1732 }
1733 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1734
1735 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1736 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1737 rc));
1738 if ( RT_SUCCESS(rc)
1739 && pVCpu)
1740 {
1741 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1742 }
1743}
1744
1745
1746/**
1747 * Flushes the TLB using VPID.
1748 *
1749 * @returns VBox status code.
1750 * @param pVM Pointer to the VM.
1751 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1752 * enmFlush).
1753 * @param enmFlush Type of flush.
1754 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1755 * on @a enmFlush).
1756 *
1757 * @remarks Can be called with interrupts disabled.
1758 */
1759static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1760{
1761 NOREF(pVM);
1762 AssertPtr(pVM);
1763 Assert(pVM->hm.s.vmx.fVpid);
1764
1765 uint64_t au64Descriptor[2];
1766 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1767 {
1768 au64Descriptor[0] = 0;
1769 au64Descriptor[1] = 0;
1770 }
1771 else
1772 {
1773 AssertPtr(pVCpu);
1774 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1775 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1776 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1777 au64Descriptor[1] = GCPtr;
1778 }
1779
1780 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1781 AssertMsg(rc == VINF_SUCCESS,
1782 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1783 if ( RT_SUCCESS(rc)
1784 && pVCpu)
1785 {
1786 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1787 }
1788}
1789
1790
1791/**
1792 * Invalidates a guest page by guest virtual address. Only relevant for
1793 * EPT/VPID, otherwise there is nothing really to invalidate.
1794 *
1795 * @returns VBox status code.
1796 * @param pVM Pointer to the VM.
1797 * @param pVCpu Pointer to the VMCPU.
1798 * @param GCVirt Guest virtual address of the page to invalidate.
1799 */
1800VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1801{
1802 AssertPtr(pVM);
1803 AssertPtr(pVCpu);
1804 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1805
1806 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1807 if (!fFlushPending)
1808 {
1809 /*
1810 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1811 * See @bugref{6043} and @bugref{6177}.
1812 *
1813 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1814 * function maybe called in a loop with individual addresses.
1815 */
1816 if (pVM->hm.s.vmx.fVpid)
1817 {
1818 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1819 {
1820 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1821 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1822 }
1823 else
1824 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1825 }
1826 else if (pVM->hm.s.fNestedPaging)
1827 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1828 }
1829
1830 return VINF_SUCCESS;
1831}
1832
1833
1834/**
1835 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1836 * otherwise there is nothing really to invalidate.
1837 *
1838 * @returns VBox status code.
1839 * @param pVM Pointer to the VM.
1840 * @param pVCpu Pointer to the VMCPU.
1841 * @param GCPhys Guest physical address of the page to invalidate.
1842 */
1843VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1844{
1845 NOREF(pVM); NOREF(GCPhys);
1846 LogFlowFunc(("%RGp\n", GCPhys));
1847
1848 /*
1849 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1850 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1851 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1852 */
1853 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1854 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1855 return VINF_SUCCESS;
1856}
1857
1858
1859/**
1860 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1861 * case where neither EPT nor VPID is supported by the CPU.
1862 *
1863 * @param pVM Pointer to the VM.
1864 * @param pVCpu Pointer to the VMCPU.
1865 * @param pCpu Pointer to the global HM struct.
1866 *
1867 * @remarks Called with interrupts disabled.
1868 */
1869static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1870{
1871 AssertPtr(pVCpu);
1872 AssertPtr(pCpu);
1873 NOREF(pVM);
1874
1875 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1876
1877 Assert(pCpu->idCpu != NIL_RTCPUID);
1878 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1879 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1880 pVCpu->hm.s.fForceTLBFlush = false;
1881 return;
1882}
1883
1884
1885/**
1886 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1887 *
1888 * @param pVM Pointer to the VM.
1889 * @param pVCpu Pointer to the VMCPU.
1890 * @param pCpu Pointer to the global HM CPU struct.
1891 * @remarks All references to "ASID" in this function pertains to "VPID" in
1892 * Intel's nomenclature. The reason is, to avoid confusion in compare
1893 * statements since the host-CPU copies are named "ASID".
1894 *
1895 * @remarks Called with interrupts disabled.
1896 */
1897static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1898{
1899#ifdef VBOX_WITH_STATISTICS
1900 bool fTlbFlushed = false;
1901# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1902# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1903 if (!fTlbFlushed) \
1904 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1905 } while (0)
1906#else
1907# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1908# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1909#endif
1910
1911 AssertPtr(pVM);
1912 AssertPtr(pCpu);
1913 AssertPtr(pVCpu);
1914 Assert(pCpu->idCpu != NIL_RTCPUID);
1915
1916 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1917 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1918 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1919
1920 /*
1921 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1922 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1923 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1924 */
1925 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1926 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1927 {
1928 ++pCpu->uCurrentAsid;
1929 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1930 {
1931 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1932 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1933 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1934 }
1935
1936 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1937 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1938 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1939
1940 /*
1941 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1942 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1943 */
1944 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1945 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1946 HMVMX_SET_TAGGED_TLB_FLUSHED();
1947 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1948 }
1949
1950 /* Check for explicit TLB flushes. */
1951 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1952 {
1953 /*
1954 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1955 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1956 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1957 * but not guest-physical mappings.
1958 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1959 */
1960 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1961 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1962 HMVMX_SET_TAGGED_TLB_FLUSHED();
1963 }
1964
1965 pVCpu->hm.s.fForceTLBFlush = false;
1966 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1967
1968 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1969 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1970 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1971 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1972 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1973 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1974 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1975 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1976 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1977
1978 /* Update VMCS with the VPID. */
1979 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1980 AssertRC(rc);
1981
1982#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1983}
1984
1985
1986/**
1987 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1988 *
1989 * @returns VBox status code.
1990 * @param pVM Pointer to the VM.
1991 * @param pVCpu Pointer to the VMCPU.
1992 * @param pCpu Pointer to the global HM CPU struct.
1993 *
1994 * @remarks Called with interrupts disabled.
1995 */
1996static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1997{
1998 AssertPtr(pVM);
1999 AssertPtr(pVCpu);
2000 AssertPtr(pCpu);
2001 Assert(pCpu->idCpu != NIL_RTCPUID);
2002 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2003 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2004
2005 /*
2006 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2007 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2008 */
2009 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2010 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2011 {
2012 pVCpu->hm.s.fForceTLBFlush = true;
2013 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2014 }
2015
2016 /* Check for explicit TLB flushes. */
2017 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2018 {
2019 pVCpu->hm.s.fForceTLBFlush = true;
2020 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2021 }
2022
2023 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2024 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2025
2026 if (pVCpu->hm.s.fForceTLBFlush)
2027 {
2028 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2029 pVCpu->hm.s.fForceTLBFlush = false;
2030 }
2031}
2032
2033
2034/**
2035 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2036 *
2037 * @returns VBox status code.
2038 * @param pVM Pointer to the VM.
2039 * @param pVCpu Pointer to the VMCPU.
2040 * @param pCpu Pointer to the global HM CPU struct.
2041 *
2042 * @remarks Called with interrupts disabled.
2043 */
2044static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2045{
2046 AssertPtr(pVM);
2047 AssertPtr(pVCpu);
2048 AssertPtr(pCpu);
2049 Assert(pCpu->idCpu != NIL_RTCPUID);
2050 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2051 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2052
2053 /*
2054 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2055 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2056 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2057 */
2058 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2059 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2060 {
2061 pVCpu->hm.s.fForceTLBFlush = true;
2062 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2063 }
2064
2065 /* Check for explicit TLB flushes. */
2066 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2067 {
2068 /*
2069 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2070 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2071 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2072 */
2073 pVCpu->hm.s.fForceTLBFlush = true;
2074 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2075 }
2076
2077 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2078 if (pVCpu->hm.s.fForceTLBFlush)
2079 {
2080 ++pCpu->uCurrentAsid;
2081 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2082 {
2083 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2084 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2085 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2086 }
2087
2088 pVCpu->hm.s.fForceTLBFlush = false;
2089 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2090 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2091 if (pCpu->fFlushAsidBeforeUse)
2092 {
2093 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2094 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2095 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2096 {
2097 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2098 pCpu->fFlushAsidBeforeUse = false;
2099 }
2100 else
2101 {
2102 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2103 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2104 }
2105 }
2106 }
2107
2108 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2109 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2110 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2111 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2112 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2113 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2114 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2115
2116 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2117 AssertRC(rc);
2118}
2119
2120
2121/**
2122 * Flushes the guest TLB entry based on CPU capabilities.
2123 *
2124 * @param pVCpu Pointer to the VMCPU.
2125 * @param pCpu Pointer to the global HM CPU struct.
2126 */
2127DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2128{
2129#ifdef HMVMX_ALWAYS_FLUSH_TLB
2130 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2131#endif
2132 PVM pVM = pVCpu->CTX_SUFF(pVM);
2133 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2134 {
2135 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2136 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2137 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2138 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2139 default:
2140 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2141 break;
2142 }
2143
2144 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2145}
2146
2147
2148/**
2149 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2150 * TLB entries from the host TLB before VM-entry.
2151 *
2152 * @returns VBox status code.
2153 * @param pVM Pointer to the VM.
2154 */
2155static int hmR0VmxSetupTaggedTlb(PVM pVM)
2156{
2157 /*
2158 * Determine optimal flush type for Nested Paging.
2159 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2160 * guest execution (see hmR3InitFinalizeR0()).
2161 */
2162 if (pVM->hm.s.fNestedPaging)
2163 {
2164 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2165 {
2166 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2167 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2168 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2169 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2170 else
2171 {
2172 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2173 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2174 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2175 }
2176
2177 /* Make sure the write-back cacheable memory type for EPT is supported. */
2178 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2179 {
2180 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2181 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2182 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2183 }
2184 }
2185 else
2186 {
2187 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2188 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2189 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2190 }
2191 }
2192
2193 /*
2194 * Determine optimal flush type for VPID.
2195 */
2196 if (pVM->hm.s.vmx.fVpid)
2197 {
2198 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2199 {
2200 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2201 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2202 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2203 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2204 else
2205 {
2206 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2207 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2208 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2209 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2210 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2211 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2212 pVM->hm.s.vmx.fVpid = false;
2213 }
2214 }
2215 else
2216 {
2217 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2218 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2219 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2220 pVM->hm.s.vmx.fVpid = false;
2221 }
2222 }
2223
2224 /*
2225 * Setup the handler for flushing tagged-TLBs.
2226 */
2227 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2228 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2229 else if (pVM->hm.s.fNestedPaging)
2230 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2231 else if (pVM->hm.s.vmx.fVpid)
2232 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2233 else
2234 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2235 return VINF_SUCCESS;
2236}
2237
2238
2239/**
2240 * Sets up pin-based VM-execution controls in the VMCS.
2241 *
2242 * @returns VBox status code.
2243 * @param pVM Pointer to the VM.
2244 * @param pVCpu Pointer to the VMCPU.
2245 */
2246static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2247{
2248 AssertPtr(pVM);
2249 AssertPtr(pVCpu);
2250
2251 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2252 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2253
2254 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2255 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2256
2257 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2258 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2259
2260 /* Enable the VMX preemption timer. */
2261 if (pVM->hm.s.vmx.fUsePreemptTimer)
2262 {
2263 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2264 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2265 }
2266
2267 if ((val & zap) != val)
2268 {
2269 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2270 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2271 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2272 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2273 }
2274
2275 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2276 AssertRCReturn(rc, rc);
2277
2278 pVCpu->hm.s.vmx.u32PinCtls = val;
2279 return rc;
2280}
2281
2282
2283/**
2284 * Sets up processor-based VM-execution controls in the VMCS.
2285 *
2286 * @returns VBox status code.
2287 * @param pVM Pointer to the VM.
2288 * @param pVMCPU Pointer to the VMCPU.
2289 */
2290static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2291{
2292 AssertPtr(pVM);
2293 AssertPtr(pVCpu);
2294
2295 int rc = VERR_INTERNAL_ERROR_5;
2296 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2297 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2298
2299 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2300 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2301 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2302 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2303 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2304 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2305 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2306
2307 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2308 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2309 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2310 {
2311 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2312 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2313 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2314 }
2315
2316 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2317 if (!pVM->hm.s.fNestedPaging)
2318 {
2319 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2320 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2321 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2322 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2323 }
2324
2325 /* Use TPR shadowing if supported by the CPU. */
2326 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2327 {
2328 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2329 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2330 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2331 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2332 AssertRCReturn(rc, rc);
2333
2334 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2335 /* CR8 writes cause a VM-exit based on TPR threshold. */
2336 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2337 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2338 }
2339 else
2340 {
2341 /*
2342 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2343 * Set this control only for 64-bit guests.
2344 */
2345 if (pVM->hm.s.fAllow64BitGuests)
2346 {
2347 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2348 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2349 }
2350 }
2351
2352 /* Use MSR-bitmaps if supported by the CPU. */
2353 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2354 {
2355 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2356
2357 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2358 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2359 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2360 AssertRCReturn(rc, rc);
2361
2362 /*
2363 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2364 * automatically using dedicated fields in the VMCS.
2365 */
2366 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2367 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2368 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2369 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2370 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2371
2372#if HC_ARCH_BITS == 64
2373 /*
2374 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2375 */
2376 if (pVM->hm.s.fAllow64BitGuests)
2377 {
2378 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2379 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2380 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2381 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2382 }
2383#endif
2384 }
2385
2386 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2387 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2388 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2389
2390 if ((val & zap) != val)
2391 {
2392 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2393 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2394 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2395 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2396 }
2397
2398 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2399 AssertRCReturn(rc, rc);
2400
2401 pVCpu->hm.s.vmx.u32ProcCtls = val;
2402
2403 /*
2404 * Secondary processor-based VM-execution controls.
2405 */
2406 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2407 {
2408 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2409 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2410
2411 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2412 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2413
2414 if (pVM->hm.s.fNestedPaging)
2415 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2416 else
2417 {
2418 /*
2419 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2420 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2421 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2422 */
2423 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2424 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2425 }
2426
2427 if (pVM->hm.s.vmx.fVpid)
2428 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2429
2430 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2431 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2432
2433 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2434 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2435 * done dynamically. */
2436 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2437 {
2438 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2439 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2440 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2441 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2442 AssertRCReturn(rc, rc);
2443 }
2444
2445 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2446 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2447
2448 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2449 && pVM->hm.s.vmx.cPleGapTicks
2450 && pVM->hm.s.vmx.cPleWindowTicks)
2451 {
2452 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2453
2454 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2455 AssertRCReturn(rc, rc);
2456
2457 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2458 AssertRCReturn(rc, rc);
2459 }
2460
2461 if ((val & zap) != val)
2462 {
2463 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2464 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2465 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2466 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2467 }
2468
2469 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2470 AssertRCReturn(rc, rc);
2471
2472 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2473 }
2474 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2475 {
2476 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2477 "available\n"));
2478 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2479 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2480 }
2481
2482 return VINF_SUCCESS;
2483}
2484
2485
2486/**
2487 * Sets up miscellaneous (everything other than Pin & Processor-based
2488 * VM-execution) control fields in the VMCS.
2489 *
2490 * @returns VBox status code.
2491 * @param pVM Pointer to the VM.
2492 * @param pVCpu Pointer to the VMCPU.
2493 */
2494static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2495{
2496 NOREF(pVM);
2497 AssertPtr(pVM);
2498 AssertPtr(pVCpu);
2499
2500 int rc = VERR_GENERAL_FAILURE;
2501
2502 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2503#if 0
2504 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2505 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2506 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2507
2508 /*
2509 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2510 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2511 * We thus use the exception bitmap to control it rather than use both.
2512 */
2513 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2514 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2515
2516 /** @todo Explore possibility of using IO-bitmaps. */
2517 /* All IO & IOIO instructions cause VM-exits. */
2518 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2519 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2520
2521 /* Initialize the MSR-bitmap area. */
2522 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2523 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2524 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2525#endif
2526
2527 /* Setup MSR auto-load/store area. */
2528 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2529 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2530 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2531 AssertRCReturn(rc, rc);
2532 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2533 AssertRCReturn(rc, rc);
2534
2535 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2536 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2537 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2538 AssertRCReturn(rc, rc);
2539
2540 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2541 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2542 AssertRCReturn(rc, rc);
2543
2544 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2545#if 0
2546 /* Setup debug controls */
2547 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2548 AssertRCReturn(rc, rc);
2549 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2550 AssertRCReturn(rc, rc);
2551#endif
2552
2553 return rc;
2554}
2555
2556
2557/**
2558 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2559 *
2560 * @returns VBox status code.
2561 * @param pVM Pointer to the VM.
2562 * @param pVCpu Pointer to the VMCPU.
2563 */
2564static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2565{
2566 AssertPtr(pVM);
2567 AssertPtr(pVCpu);
2568
2569 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2570
2571 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2572
2573 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2574 if (!pVM->hm.s.fNestedPaging)
2575 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2576
2577 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2578 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2579 AssertRCReturn(rc, rc);
2580 return rc;
2581}
2582
2583
2584/**
2585 * Sets up the initial guest-state mask. The guest-state mask is consulted
2586 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2587 * for the nested virtualization case (as it would cause a VM-exit).
2588 *
2589 * @param pVCpu Pointer to the VMCPU.
2590 */
2591static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2592{
2593 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2594 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2595 return VINF_SUCCESS;
2596}
2597
2598
2599/**
2600 * Does per-VM VT-x initialization.
2601 *
2602 * @returns VBox status code.
2603 * @param pVM Pointer to the VM.
2604 */
2605VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2606{
2607 LogFlowFunc(("pVM=%p\n", pVM));
2608
2609 int rc = hmR0VmxStructsAlloc(pVM);
2610 if (RT_FAILURE(rc))
2611 {
2612 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2613 return rc;
2614 }
2615
2616 return VINF_SUCCESS;
2617}
2618
2619
2620/**
2621 * Does per-VM VT-x termination.
2622 *
2623 * @returns VBox status code.
2624 * @param pVM Pointer to the VM.
2625 */
2626VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2627{
2628 LogFlowFunc(("pVM=%p\n", pVM));
2629
2630#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2631 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2632 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2633#endif
2634 hmR0VmxStructsFree(pVM);
2635 return VINF_SUCCESS;
2636}
2637
2638
2639/**
2640 * Sets up the VM for execution under VT-x.
2641 * This function is only called once per-VM during initialization.
2642 *
2643 * @returns VBox status code.
2644 * @param pVM Pointer to the VM.
2645 */
2646VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2647{
2648 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2649 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2650
2651 LogFlowFunc(("pVM=%p\n", pVM));
2652
2653 /*
2654 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2655 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2656 */
2657 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2658 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2659 || !pVM->hm.s.vmx.pRealModeTSS))
2660 {
2661 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2662 return VERR_INTERNAL_ERROR;
2663 }
2664
2665 /* Initialize these always, see hmR3InitFinalizeR0().*/
2666 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2667 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2668
2669 /* Setup the tagged-TLB flush handlers. */
2670 int rc = hmR0VmxSetupTaggedTlb(pVM);
2671 if (RT_FAILURE(rc))
2672 {
2673 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2674 return rc;
2675 }
2676
2677 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2678 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2679#if HC_ARCH_BITS == 64
2680 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2681 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2682 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2683 {
2684 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2685 }
2686#endif
2687
2688 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2689 {
2690 PVMCPU pVCpu = &pVM->aCpus[i];
2691 AssertPtr(pVCpu);
2692 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2693
2694 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2695 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2696
2697 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2698 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2699 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2700
2701 /* Set revision dword at the beginning of the VMCS structure. */
2702 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2703
2704 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2705 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2706 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2707 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2708
2709 /* Load this VMCS as the current VMCS. */
2710 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2711 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2712 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2713
2714 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2715 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2716 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2717
2718 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2719 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2720 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2721
2722 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2723 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2724 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2725
2726 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2727 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2728 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2729
2730 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2731 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2732 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2733
2734#if HC_ARCH_BITS == 32
2735 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2736 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2737 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2738#endif
2739
2740 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2741 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2742 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2743 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2744
2745 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2746
2747 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2748 }
2749
2750 return VINF_SUCCESS;
2751}
2752
2753
2754/**
2755 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2756 * the VMCS.
2757 *
2758 * @returns VBox status code.
2759 * @param pVM Pointer to the VM.
2760 * @param pVCpu Pointer to the VMCPU.
2761 */
2762DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2763{
2764 NOREF(pVM); NOREF(pVCpu);
2765
2766 RTCCUINTREG uReg = ASMGetCR0();
2767 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2768 AssertRCReturn(rc, rc);
2769
2770 uReg = ASMGetCR3();
2771 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2772 AssertRCReturn(rc, rc);
2773
2774 uReg = ASMGetCR4();
2775 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2776 AssertRCReturn(rc, rc);
2777 return rc;
2778}
2779
2780
2781#if HC_ARCH_BITS == 64
2782/**
2783 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2784 * requirements. See hmR0VmxSaveHostSegmentRegs().
2785 */
2786# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2787 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2788 { \
2789 bool fValidSelector = true; \
2790 if ((selValue) & X86_SEL_LDT) \
2791 { \
2792 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2793 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2794 } \
2795 if (fValidSelector) \
2796 { \
2797 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2798 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2799 } \
2800 (selValue) = 0; \
2801 }
2802#endif
2803
2804
2805/**
2806 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2807 * the host-state area in the VMCS.
2808 *
2809 * @returns VBox status code.
2810 * @param pVM Pointer to the VM.
2811 * @param pVCpu Pointer to the VMCPU.
2812 */
2813DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2814{
2815 int rc = VERR_INTERNAL_ERROR_5;
2816
2817#if HC_ARCH_BITS == 64
2818 /*
2819 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2820 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2821 */
2822 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2823 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2824#endif
2825
2826 /*
2827 * Host DS, ES, FS and GS segment registers.
2828 */
2829#if HC_ARCH_BITS == 64
2830 RTSEL uSelDS = ASMGetDS();
2831 RTSEL uSelES = ASMGetES();
2832 RTSEL uSelFS = ASMGetFS();
2833 RTSEL uSelGS = ASMGetGS();
2834#else
2835 RTSEL uSelDS = 0;
2836 RTSEL uSelES = 0;
2837 RTSEL uSelFS = 0;
2838 RTSEL uSelGS = 0;
2839#endif
2840
2841 /* Recalculate which host-state bits need to be manually restored. */
2842 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2843
2844 /*
2845 * Host CS and SS segment registers.
2846 */
2847 RTSEL uSelCS = ASMGetCS();
2848 RTSEL uSelSS = ASMGetSS();
2849
2850 /*
2851 * Host TR segment register.
2852 */
2853 RTSEL uSelTR = ASMGetTR();
2854
2855#if HC_ARCH_BITS == 64
2856 /*
2857 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2858 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2859 */
2860 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2861 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2862 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2863 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2864# undef VMXLOCAL_ADJUST_HOST_SEG
2865#endif
2866
2867 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2868 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2869 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2870 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2871 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2872 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2873 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2874 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2875 Assert(uSelCS);
2876 Assert(uSelTR);
2877
2878 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2879#if 0
2880 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2881 Assert(uSelSS != 0);
2882#endif
2883
2884 /* Write these host selector fields into the host-state area in the VMCS. */
2885 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2886 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2887#if HC_ARCH_BITS == 64
2888 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2889 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2890 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2891 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2892#endif
2893 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2894
2895 /*
2896 * Host GDTR and IDTR.
2897 */
2898 RTGDTR Gdtr;
2899 RTIDTR Idtr;
2900 RT_ZERO(Gdtr);
2901 RT_ZERO(Idtr);
2902 ASMGetGDTR(&Gdtr);
2903 ASMGetIDTR(&Idtr);
2904 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2905 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2906
2907#if HC_ARCH_BITS == 64
2908 /*
2909 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2910 * maximum limit (0xffff) on every VM-exit.
2911 */
2912 if (Gdtr.cbGdt != 0xffff)
2913 {
2914 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2915 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2916 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2917 }
2918
2919 /*
2920 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2921 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2922 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2923 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2924 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
2925 * hosts where we are pretty sure it won't cause trouble.
2926 */
2927# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2928 if (Idtr.cbIdt < 0x0fff)
2929# else
2930 if (Idtr.cbIdt != 0xffff)
2931# endif
2932 {
2933 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2934 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2935 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2936 }
2937#endif
2938
2939 /*
2940 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2941 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2942 */
2943 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
2944 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
2945 VERR_VMX_INVALID_HOST_STATE);
2946
2947 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2948#if HC_ARCH_BITS == 64
2949 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
2950
2951 /*
2952 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2953 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2954 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2955 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2956 *
2957 * [1] See Intel spec. 3.5 "System Descriptor Types".
2958 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2959 */
2960 Assert(pDesc->System.u4Type == 11);
2961 if ( pDesc->System.u16LimitLow != 0x67
2962 || pDesc->System.u4LimitHigh)
2963 {
2964 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2965 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
2966 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
2967 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
2968 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2969
2970 /* Store the GDTR here as we need it while restoring TR. */
2971 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2972 }
2973#else
2974 uintptr_t uTRBase = X86DESC_BASE(pDesc);
2975#endif
2976 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2977 AssertRCReturn(rc, rc);
2978
2979 /*
2980 * Host FS base and GS base.
2981 */
2982#if HC_ARCH_BITS == 64
2983 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2984 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2985 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2986 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2987
2988 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2989 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2990 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2991 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2992 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2993#endif
2994 return rc;
2995}
2996
2997
2998/**
2999 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3000 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3001 * the host after every successful VM-exit.
3002 *
3003 * @returns VBox status code.
3004 * @param pVM Pointer to the VM.
3005 * @param pVCpu Pointer to the VMCPU.
3006 *
3007 * @remarks No-long-jump zone!!!
3008 */
3009DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3010{
3011 NOREF(pVM);
3012
3013 AssertPtr(pVCpu);
3014 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3015
3016 int rc = VINF_SUCCESS;
3017#if HC_ARCH_BITS == 64
3018 if (pVM->hm.s.fAllow64BitGuests)
3019 hmR0VmxLazySaveHostMsrs(pVCpu);
3020#endif
3021
3022 /*
3023 * Host Sysenter MSRs.
3024 */
3025 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3026 AssertRCReturn(rc, rc);
3027#if HC_ARCH_BITS == 32
3028 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3029 AssertRCReturn(rc, rc);
3030 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3031#else
3032 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3033 AssertRCReturn(rc, rc);
3034 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3035#endif
3036 AssertRCReturn(rc, rc);
3037
3038 /*
3039 * Host EFER MSR.
3040 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3041 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3042 */
3043 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3044 {
3045 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3046 AssertRCReturn(rc, rc);
3047 }
3048
3049 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3050 * hmR0VmxLoadGuestExitCtls() !! */
3051
3052 return rc;
3053}
3054
3055
3056/**
3057 * Figures out if we need to swap the EFER MSR which is
3058 * particularly expensive.
3059 *
3060 * We check all relevant bits. For now, that's everything
3061 * besides LMA/LME, as these two bits are handled by VM-entry,
3062 * see hmR0VmxLoadGuestExitCtls() and
3063 * hmR0VMxLoadGuestEntryCtls().
3064 *
3065 * @returns true if we need to load guest EFER, false otherwise.
3066 * @param pVCpu Pointer to the VMCPU.
3067 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3068 * out-of-sync. Make sure to update the required fields
3069 * before using them.
3070 *
3071 * @remarks Requires EFER, CR4.
3072 * @remarks No-long-jump zone!!!
3073 */
3074static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3075{
3076#ifdef HMVMX_ALWAYS_SWAP_EFER
3077 return true;
3078#endif
3079
3080#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3081 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3082 if (CPUMIsGuestInLongMode(pVCpu))
3083 return false;
3084#endif
3085
3086 PVM pVM = pVCpu->CTX_SUFF(pVM);
3087 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3088 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3089
3090 /*
3091 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3092 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3093 */
3094 if ( CPUMIsGuestInLongMode(pVCpu)
3095 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3096 {
3097 return true;
3098 }
3099
3100 /*
3101 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3102 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3103 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3104 */
3105 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3106 && (pMixedCtx->cr0 & X86_CR0_PG)
3107 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3108 {
3109 /* Assert that host is PAE capable. */
3110 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3111 return true;
3112 }
3113
3114 /** @todo Check the latest Intel spec. for any other bits,
3115 * like SMEP/SMAP? */
3116 return false;
3117}
3118
3119
3120/**
3121 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3122 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3123 * controls".
3124 *
3125 * @returns VBox status code.
3126 * @param pVCpu Pointer to the VMCPU.
3127 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3128 * out-of-sync. Make sure to update the required fields
3129 * before using them.
3130 *
3131 * @remarks Requires EFER.
3132 * @remarks No-long-jump zone!!!
3133 */
3134DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3135{
3136 int rc = VINF_SUCCESS;
3137 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3138 {
3139 PVM pVM = pVCpu->CTX_SUFF(pVM);
3140 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3141 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3142
3143 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3144 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3145
3146 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3147 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3148 {
3149 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3150 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3151 }
3152 else
3153 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3154
3155 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3156 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3157 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3158 {
3159 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3160 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3161 }
3162
3163 /*
3164 * The following should -not- be set (since we're not in SMM mode):
3165 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3166 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3167 */
3168
3169 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3170 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3171
3172 if ((val & zap) != val)
3173 {
3174 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3175 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3176 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3177 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3178 }
3179
3180 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3181 AssertRCReturn(rc, rc);
3182
3183 pVCpu->hm.s.vmx.u32EntryCtls = val;
3184 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3185 }
3186 return rc;
3187}
3188
3189
3190/**
3191 * Sets up the VM-exit controls in the VMCS.
3192 *
3193 * @returns VBox status code.
3194 * @param pVM Pointer to the VM.
3195 * @param pVCpu Pointer to the VMCPU.
3196 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3197 * out-of-sync. Make sure to update the required fields
3198 * before using them.
3199 *
3200 * @remarks Requires EFER.
3201 */
3202DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3203{
3204 NOREF(pMixedCtx);
3205
3206 int rc = VINF_SUCCESS;
3207 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3208 {
3209 PVM pVM = pVCpu->CTX_SUFF(pVM);
3210 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3211 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3212
3213 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3214 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3215
3216 /*
3217 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3218 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3219 */
3220#if HC_ARCH_BITS == 64
3221 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3222 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3223#else
3224 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3225 {
3226 /* The switcher returns to long mode, EFER is managed by the switcher. */
3227 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3228 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3229 }
3230 else
3231 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3232#endif
3233
3234 /* If the newer VMCS fields for managing EFER exists, use it. */
3235 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3236 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3237 {
3238 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3239 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3240 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3241 }
3242
3243 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3244 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3245
3246 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3247 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3248 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3249
3250 if ( pVM->hm.s.vmx.fUsePreemptTimer
3251 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3252 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3253
3254 if ((val & zap) != val)
3255 {
3256 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3257 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3258 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3259 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3260 }
3261
3262 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3263 AssertRCReturn(rc, rc);
3264
3265 pVCpu->hm.s.vmx.u32ExitCtls = val;
3266 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3267 }
3268 return rc;
3269}
3270
3271
3272/**
3273 * Loads the guest APIC and related state.
3274 *
3275 * @returns VBox status code.
3276 * @param pVM Pointer to the VM.
3277 * @param pVCpu Pointer to the VMCPU.
3278 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3279 * out-of-sync. Make sure to update the required fields
3280 * before using them.
3281 */
3282DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3283{
3284 NOREF(pMixedCtx);
3285
3286 int rc = VINF_SUCCESS;
3287 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3288 {
3289 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3290 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3291 {
3292 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3293
3294 bool fPendingIntr = false;
3295 uint8_t u8Tpr = 0;
3296 uint8_t u8PendingIntr = 0;
3297 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3298 AssertRCReturn(rc, rc);
3299
3300 /*
3301 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3302 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3303 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3304 * the interrupt when we VM-exit for other reasons.
3305 */
3306 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3307 uint32_t u32TprThreshold = 0;
3308 if (fPendingIntr)
3309 {
3310 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3311 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3312 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3313 if (u8PendingPriority <= u8TprPriority)
3314 u32TprThreshold = u8PendingPriority;
3315 else
3316 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3317 }
3318 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3319
3320 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3321 AssertRCReturn(rc, rc);
3322 }
3323
3324 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3325 }
3326 return rc;
3327}
3328
3329
3330/**
3331 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3332 *
3333 * @returns Guest's interruptibility-state.
3334 * @param pVCpu Pointer to the VMCPU.
3335 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3336 * out-of-sync. Make sure to update the required fields
3337 * before using them.
3338 *
3339 * @remarks No-long-jump zone!!!
3340 */
3341DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3342{
3343 /*
3344 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3345 */
3346 uint32_t uIntrState = 0;
3347 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3348 {
3349 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3350 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3351 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3352 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3353 {
3354 if (pMixedCtx->eflags.Bits.u1IF)
3355 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3356 else
3357 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3358 }
3359 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3360 }
3361
3362 /*
3363 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3364 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3365 * setting this would block host-NMIs and IRET will not clear the blocking.
3366 *
3367 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3368 */
3369 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3370 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3371 {
3372 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3373 }
3374
3375 return uIntrState;
3376}
3377
3378
3379/**
3380 * Loads the guest's interruptibility-state into the guest-state area in the
3381 * VMCS.
3382 *
3383 * @returns VBox status code.
3384 * @param pVCpu Pointer to the VMCPU.
3385 * @param uIntrState The interruptibility-state to set.
3386 */
3387static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3388{
3389 NOREF(pVCpu);
3390 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3391 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3392 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3393 AssertRCReturn(rc, rc);
3394 return rc;
3395}
3396
3397
3398/**
3399 * Loads the exception intercepts required for guest execution in the VMCS.
3400 *
3401 * @returns VBox status code.
3402 * @param pVCpu Pointer to the VMCPU.
3403 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3404 * out-of-sync. Make sure to update the required fields
3405 * before using them.
3406 */
3407static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3408{
3409 NOREF(pMixedCtx);
3410 int rc = VINF_SUCCESS;
3411 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3412 {
3413 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3414 if (pVCpu->hm.s.fGIMTrapXcptUD)
3415 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3416 else
3417 {
3418#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3419 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3420#endif
3421 }
3422
3423 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3424 AssertRCReturn(rc, rc);
3425
3426 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3427 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3428 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3429 }
3430 return rc;
3431}
3432
3433
3434/**
3435 * Loads the guest's RIP into the guest-state area in the VMCS.
3436 *
3437 * @returns VBox status code.
3438 * @param pVCpu Pointer to the VMCPU.
3439 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3440 * out-of-sync. Make sure to update the required fields
3441 * before using them.
3442 *
3443 * @remarks No-long-jump zone!!!
3444 */
3445static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3446{
3447 int rc = VINF_SUCCESS;
3448 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3449 {
3450 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3451 AssertRCReturn(rc, rc);
3452
3453 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3454 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3455 HMCPU_CF_VALUE(pVCpu)));
3456 }
3457 return rc;
3458}
3459
3460
3461/**
3462 * Loads the guest's RSP into the guest-state area in the VMCS.
3463 *
3464 * @returns VBox status code.
3465 * @param pVCpu Pointer to the VMCPU.
3466 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3467 * out-of-sync. Make sure to update the required fields
3468 * before using them.
3469 *
3470 * @remarks No-long-jump zone!!!
3471 */
3472static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3473{
3474 int rc = VINF_SUCCESS;
3475 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3476 {
3477 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3478 AssertRCReturn(rc, rc);
3479
3480 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3481 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3482 }
3483 return rc;
3484}
3485
3486
3487/**
3488 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3489 *
3490 * @returns VBox status code.
3491 * @param pVCpu Pointer to the VMCPU.
3492 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3493 * out-of-sync. Make sure to update the required fields
3494 * before using them.
3495 *
3496 * @remarks No-long-jump zone!!!
3497 */
3498static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3499{
3500 int rc = VINF_SUCCESS;
3501 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3502 {
3503 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3504 Let us assert it as such and use 32-bit VMWRITE. */
3505 Assert(!(pMixedCtx->rflags.u64 >> 32));
3506 X86EFLAGS Eflags = pMixedCtx->eflags;
3507 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3508 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3509 * These will never be cleared/set, unless some other part of the VMM
3510 * code is buggy - in which case we're better of finding and fixing
3511 * those bugs than hiding them. */
3512 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3513 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3514 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3515 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3516
3517 /*
3518 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3519 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3520 */
3521 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3522 {
3523 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3524 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3525 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3526 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3527 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3528 }
3529
3530 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3531 AssertRCReturn(rc, rc);
3532
3533 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3534 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3535 }
3536 return rc;
3537}
3538
3539
3540/**
3541 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3542 *
3543 * @returns VBox status code.
3544 * @param pVCpu Pointer to the VMCPU.
3545 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3546 * out-of-sync. Make sure to update the required fields
3547 * before using them.
3548 *
3549 * @remarks No-long-jump zone!!!
3550 */
3551DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3552{
3553 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3554 AssertRCReturn(rc, rc);
3555 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3556 AssertRCReturn(rc, rc);
3557 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3558 AssertRCReturn(rc, rc);
3559 return rc;
3560}
3561
3562
3563/**
3564 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3565 * CR0 is partially shared with the host and we have to consider the FPU bits.
3566 *
3567 * @returns VBox status code.
3568 * @param pVM Pointer to the VM.
3569 * @param pVCpu Pointer to the VMCPU.
3570 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3571 * out-of-sync. Make sure to update the required fields
3572 * before using them.
3573 *
3574 * @remarks No-long-jump zone!!!
3575 */
3576static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3577{
3578 /*
3579 * Guest CR0.
3580 * Guest FPU.
3581 */
3582 int rc = VINF_SUCCESS;
3583 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3584 {
3585 Assert(!(pMixedCtx->cr0 >> 32));
3586 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3587 PVM pVM = pVCpu->CTX_SUFF(pVM);
3588
3589 /* The guest's view (read access) of its CR0 is unblemished. */
3590 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3591 AssertRCReturn(rc, rc);
3592 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3593
3594 /* Setup VT-x's view of the guest CR0. */
3595 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3596 if (pVM->hm.s.fNestedPaging)
3597 {
3598 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3599 {
3600 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3601 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3602 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3603 }
3604 else
3605 {
3606 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3607 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3608 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3609 }
3610
3611 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3612 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3613 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3614
3615 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3616 AssertRCReturn(rc, rc);
3617 }
3618 else
3619 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3620
3621 /*
3622 * Guest FPU bits.
3623 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3624 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3625 */
3626 u32GuestCR0 |= X86_CR0_NE;
3627 bool fInterceptNM = false;
3628 if (CPUMIsGuestFPUStateActive(pVCpu))
3629 {
3630 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3631 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3632 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3633 }
3634 else
3635 {
3636 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3637 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3638 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3639 }
3640
3641 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3642 bool fInterceptMF = false;
3643 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3644 fInterceptMF = true;
3645
3646 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3647 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3648 {
3649 Assert(PDMVmmDevHeapIsEnabled(pVM));
3650 Assert(pVM->hm.s.vmx.pRealModeTSS);
3651 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3652 fInterceptNM = true;
3653 fInterceptMF = true;
3654 }
3655 else
3656 {
3657 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3658 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3659 }
3660 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3661
3662 if (fInterceptNM)
3663 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3664 else
3665 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3666
3667 if (fInterceptMF)
3668 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3669 else
3670 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3671
3672 /* Additional intercepts for debugging, define these yourself explicitly. */
3673#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3674 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3675 | RT_BIT(X86_XCPT_BP)
3676 | RT_BIT(X86_XCPT_DB)
3677 | RT_BIT(X86_XCPT_DE)
3678 | RT_BIT(X86_XCPT_NM)
3679 | RT_BIT(X86_XCPT_TS)
3680 | RT_BIT(X86_XCPT_UD)
3681 | RT_BIT(X86_XCPT_NP)
3682 | RT_BIT(X86_XCPT_SS)
3683 | RT_BIT(X86_XCPT_GP)
3684 | RT_BIT(X86_XCPT_PF)
3685 | RT_BIT(X86_XCPT_MF)
3686 ;
3687#elif defined(HMVMX_ALWAYS_TRAP_PF)
3688 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3689#endif
3690
3691 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3692
3693 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3694 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3695 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3696 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3697 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3698 else
3699 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3700
3701 u32GuestCR0 |= uSetCR0;
3702 u32GuestCR0 &= uZapCR0;
3703 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3704
3705 /* Write VT-x's view of the guest CR0 into the VMCS. */
3706 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3707 AssertRCReturn(rc, rc);
3708 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3709 uZapCR0));
3710
3711 /*
3712 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3713 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3714 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3715 */
3716 uint32_t u32CR0Mask = 0;
3717 u32CR0Mask = X86_CR0_PE
3718 | X86_CR0_NE
3719 | X86_CR0_WP
3720 | X86_CR0_PG
3721 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3722 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3723 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3724
3725 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3726 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3727 * and @bugref{6944}. */
3728#if 0
3729 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3730 u32CR0Mask &= ~X86_CR0_PE;
3731#endif
3732 if (pVM->hm.s.fNestedPaging)
3733 u32CR0Mask &= ~X86_CR0_WP;
3734
3735 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3736 if (fInterceptNM)
3737 {
3738 u32CR0Mask |= X86_CR0_TS
3739 | X86_CR0_MP;
3740 }
3741
3742 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3743 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3744 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3745 AssertRCReturn(rc, rc);
3746 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3747
3748 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3749 }
3750 return rc;
3751}
3752
3753
3754/**
3755 * Loads the guest control registers (CR3, CR4) into the guest-state area
3756 * in the VMCS.
3757 *
3758 * @returns VBox status code.
3759 * @param pVM Pointer to the VM.
3760 * @param pVCpu Pointer to the VMCPU.
3761 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3762 * out-of-sync. Make sure to update the required fields
3763 * before using them.
3764 *
3765 * @remarks No-long-jump zone!!!
3766 */
3767static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3768{
3769 int rc = VINF_SUCCESS;
3770 PVM pVM = pVCpu->CTX_SUFF(pVM);
3771
3772 /*
3773 * Guest CR2.
3774 * It's always loaded in the assembler code. Nothing to do here.
3775 */
3776
3777 /*
3778 * Guest CR3.
3779 */
3780 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3781 {
3782 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3783 if (pVM->hm.s.fNestedPaging)
3784 {
3785 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3786
3787 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3788 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3789 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3790 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3791
3792 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3793 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3794 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3795
3796 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3797 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3798 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3799 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3800
3801 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3802 AssertRCReturn(rc, rc);
3803 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3804
3805 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3806 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3807 {
3808 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3809 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3810 {
3811 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3812 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3813 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3814 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3815 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3816 }
3817
3818 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3819 have Unrestricted Execution to handle the guest when it's not using paging. */
3820 GCPhysGuestCR3 = pMixedCtx->cr3;
3821 }
3822 else
3823 {
3824 /*
3825 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3826 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3827 * EPT takes care of translating it to host-physical addresses.
3828 */
3829 RTGCPHYS GCPhys;
3830 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3831 Assert(PDMVmmDevHeapIsEnabled(pVM));
3832
3833 /* We obtain it here every time as the guest could have relocated this PCI region. */
3834 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3835 AssertRCReturn(rc, rc);
3836
3837 GCPhysGuestCR3 = GCPhys;
3838 }
3839
3840 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3841 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3842 }
3843 else
3844 {
3845 /* Non-nested paging case, just use the hypervisor's CR3. */
3846 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3847
3848 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3849 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3850 }
3851 AssertRCReturn(rc, rc);
3852
3853 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3854 }
3855
3856 /*
3857 * Guest CR4.
3858 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3859 */
3860 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3861 {
3862 Assert(!(pMixedCtx->cr4 >> 32));
3863 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3864
3865 /* The guest's view of its CR4 is unblemished. */
3866 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3867 AssertRCReturn(rc, rc);
3868 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3869
3870 /* Setup VT-x's view of the guest CR4. */
3871 /*
3872 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3873 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3874 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3875 */
3876 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3877 {
3878 Assert(pVM->hm.s.vmx.pRealModeTSS);
3879 Assert(PDMVmmDevHeapIsEnabled(pVM));
3880 u32GuestCR4 &= ~X86_CR4_VME;
3881 }
3882
3883 if (pVM->hm.s.fNestedPaging)
3884 {
3885 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3886 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3887 {
3888 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3889 u32GuestCR4 |= X86_CR4_PSE;
3890 /* Our identity mapping is a 32-bit page directory. */
3891 u32GuestCR4 &= ~X86_CR4_PAE;
3892 }
3893 /* else use guest CR4.*/
3894 }
3895 else
3896 {
3897 /*
3898 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3899 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3900 */
3901 switch (pVCpu->hm.s.enmShadowMode)
3902 {
3903 case PGMMODE_REAL: /* Real-mode. */
3904 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3905 case PGMMODE_32_BIT: /* 32-bit paging. */
3906 {
3907 u32GuestCR4 &= ~X86_CR4_PAE;
3908 break;
3909 }
3910
3911 case PGMMODE_PAE: /* PAE paging. */
3912 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3913 {
3914 u32GuestCR4 |= X86_CR4_PAE;
3915 break;
3916 }
3917
3918 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3919 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3920#ifdef VBOX_ENABLE_64_BITS_GUESTS
3921 break;
3922#endif
3923 default:
3924 AssertFailed();
3925 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3926 }
3927 }
3928
3929 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3930 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3931 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3932 u32GuestCR4 |= uSetCR4;
3933 u32GuestCR4 &= uZapCR4;
3934
3935 /* Write VT-x's view of the guest CR4 into the VMCS. */
3936 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
3937 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3938 AssertRCReturn(rc, rc);
3939
3940 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3941 uint32_t u32CR4Mask = X86_CR4_VME
3942 | X86_CR4_PAE
3943 | X86_CR4_PGE
3944 | X86_CR4_PSE
3945 | X86_CR4_VMXE;
3946 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
3947 u32CR4Mask |= X86_CR4_OSXSAVE;
3948 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3949 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3950 AssertRCReturn(rc, rc);
3951
3952 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
3953 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
3954
3955 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3956 }
3957 return rc;
3958}
3959
3960
3961/**
3962 * Loads the guest debug registers into the guest-state area in the VMCS.
3963 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3964 *
3965 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3966 *
3967 * @returns VBox status code.
3968 * @param pVCpu Pointer to the VMCPU.
3969 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3970 * out-of-sync. Make sure to update the required fields
3971 * before using them.
3972 *
3973 * @remarks No-long-jump zone!!!
3974 */
3975static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3976{
3977 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3978 return VINF_SUCCESS;
3979
3980#ifdef VBOX_STRICT
3981 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3982 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3983 {
3984 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3985 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3986 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3987 }
3988#endif
3989
3990 int rc;
3991 PVM pVM = pVCpu->CTX_SUFF(pVM);
3992 bool fInterceptDB = false;
3993 bool fInterceptMovDRx = false;
3994 if ( pVCpu->hm.s.fSingleInstruction
3995 || DBGFIsStepping(pVCpu))
3996 {
3997 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3998 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3999 {
4000 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4001 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4002 AssertRCReturn(rc, rc);
4003 Assert(fInterceptDB == false);
4004 }
4005 else
4006 {
4007 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4008 pVCpu->hm.s.fClearTrapFlag = true;
4009 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4010 fInterceptDB = true;
4011 }
4012 }
4013
4014 if ( fInterceptDB
4015 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4016 {
4017 /*
4018 * Use the combined guest and host DRx values found in the hypervisor
4019 * register set because the debugger has breakpoints active or someone
4020 * is single stepping on the host side without a monitor trap flag.
4021 *
4022 * Note! DBGF expects a clean DR6 state before executing guest code.
4023 */
4024#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4025 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4026 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4027 {
4028 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4029 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4030 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4031 }
4032 else
4033#endif
4034 if (!CPUMIsHyperDebugStateActive(pVCpu))
4035 {
4036 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4037 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4038 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4039 }
4040
4041 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4042 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4043 AssertRCReturn(rc, rc);
4044
4045 pVCpu->hm.s.fUsingHyperDR7 = true;
4046 fInterceptDB = true;
4047 fInterceptMovDRx = true;
4048 }
4049 else
4050 {
4051 /*
4052 * If the guest has enabled debug registers, we need to load them prior to
4053 * executing guest code so they'll trigger at the right time.
4054 */
4055 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4056 {
4057#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4058 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4059 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4060 {
4061 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4062 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4063 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4064 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4065 }
4066 else
4067#endif
4068 if (!CPUMIsGuestDebugStateActive(pVCpu))
4069 {
4070 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4071 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4072 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4073 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4074 }
4075 Assert(!fInterceptDB);
4076 Assert(!fInterceptMovDRx);
4077 }
4078 /*
4079 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4080 * must intercept #DB in order to maintain a correct DR6 guest value.
4081 */
4082#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4083 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4084 && !CPUMIsGuestDebugStateActive(pVCpu))
4085#else
4086 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4087#endif
4088 {
4089 fInterceptMovDRx = true;
4090 fInterceptDB = true;
4091 }
4092
4093 /* Update guest DR7. */
4094 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4095 AssertRCReturn(rc, rc);
4096
4097 pVCpu->hm.s.fUsingHyperDR7 = false;
4098 }
4099
4100 /*
4101 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4102 */
4103 if ( fInterceptDB
4104 || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4105 {
4106 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4107 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
4108 }
4109 else
4110 {
4111#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4112 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4113 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
4114#endif
4115 }
4116
4117 /*
4118 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4119 */
4120 if (fInterceptMovDRx)
4121 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4122 else
4123 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4124 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4125 AssertRCReturn(rc, rc);
4126
4127 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4128 return VINF_SUCCESS;
4129}
4130
4131
4132#ifdef VBOX_STRICT
4133/**
4134 * Strict function to validate segment registers.
4135 *
4136 * @remarks ASSUMES CR0 is up to date.
4137 */
4138static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4139{
4140 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4141 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4142 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4143 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4144 && ( !CPUMIsGuestInRealModeEx(pCtx)
4145 && !CPUMIsGuestInV86ModeEx(pCtx)))
4146 {
4147 /* Protected mode checks */
4148 /* CS */
4149 Assert(pCtx->cs.Attr.n.u1Present);
4150 Assert(!(pCtx->cs.Attr.u & 0xf00));
4151 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4152 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4153 || !(pCtx->cs.Attr.n.u1Granularity));
4154 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4155 || (pCtx->cs.Attr.n.u1Granularity));
4156 /* CS cannot be loaded with NULL in protected mode. */
4157 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4158 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4159 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4160 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4161 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4162 else
4163 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4164 /* SS */
4165 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4166 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4167 if ( !(pCtx->cr0 & X86_CR0_PE)
4168 || pCtx->cs.Attr.n.u4Type == 3)
4169 {
4170 Assert(!pCtx->ss.Attr.n.u2Dpl);
4171 }
4172 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4173 {
4174 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4175 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4176 Assert(pCtx->ss.Attr.n.u1Present);
4177 Assert(!(pCtx->ss.Attr.u & 0xf00));
4178 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4179 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4180 || !(pCtx->ss.Attr.n.u1Granularity));
4181 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4182 || (pCtx->ss.Attr.n.u1Granularity));
4183 }
4184 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4185 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4186 {
4187 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4188 Assert(pCtx->ds.Attr.n.u1Present);
4189 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4190 Assert(!(pCtx->ds.Attr.u & 0xf00));
4191 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4192 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4193 || !(pCtx->ds.Attr.n.u1Granularity));
4194 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4195 || (pCtx->ds.Attr.n.u1Granularity));
4196 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4197 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4198 }
4199 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4200 {
4201 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4202 Assert(pCtx->es.Attr.n.u1Present);
4203 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4204 Assert(!(pCtx->es.Attr.u & 0xf00));
4205 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4206 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4207 || !(pCtx->es.Attr.n.u1Granularity));
4208 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4209 || (pCtx->es.Attr.n.u1Granularity));
4210 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4211 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4212 }
4213 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4214 {
4215 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4216 Assert(pCtx->fs.Attr.n.u1Present);
4217 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4218 Assert(!(pCtx->fs.Attr.u & 0xf00));
4219 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4220 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4221 || !(pCtx->fs.Attr.n.u1Granularity));
4222 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4223 || (pCtx->fs.Attr.n.u1Granularity));
4224 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4225 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4226 }
4227 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4228 {
4229 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4230 Assert(pCtx->gs.Attr.n.u1Present);
4231 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4232 Assert(!(pCtx->gs.Attr.u & 0xf00));
4233 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4234 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4235 || !(pCtx->gs.Attr.n.u1Granularity));
4236 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4237 || (pCtx->gs.Attr.n.u1Granularity));
4238 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4239 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4240 }
4241 /* 64-bit capable CPUs. */
4242# if HC_ARCH_BITS == 64
4243 Assert(!(pCtx->cs.u64Base >> 32));
4244 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4245 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4246 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4247# endif
4248 }
4249 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4250 || ( CPUMIsGuestInRealModeEx(pCtx)
4251 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4252 {
4253 /* Real and v86 mode checks. */
4254 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4255 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4256 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4257 {
4258 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4259 }
4260 else
4261 {
4262 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4263 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4264 }
4265
4266 /* CS */
4267 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4268 Assert(pCtx->cs.u32Limit == 0xffff);
4269 Assert(u32CSAttr == 0xf3);
4270 /* SS */
4271 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4272 Assert(pCtx->ss.u32Limit == 0xffff);
4273 Assert(u32SSAttr == 0xf3);
4274 /* DS */
4275 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4276 Assert(pCtx->ds.u32Limit == 0xffff);
4277 Assert(u32DSAttr == 0xf3);
4278 /* ES */
4279 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4280 Assert(pCtx->es.u32Limit == 0xffff);
4281 Assert(u32ESAttr == 0xf3);
4282 /* FS */
4283 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4284 Assert(pCtx->fs.u32Limit == 0xffff);
4285 Assert(u32FSAttr == 0xf3);
4286 /* GS */
4287 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4288 Assert(pCtx->gs.u32Limit == 0xffff);
4289 Assert(u32GSAttr == 0xf3);
4290 /* 64-bit capable CPUs. */
4291# if HC_ARCH_BITS == 64
4292 Assert(!(pCtx->cs.u64Base >> 32));
4293 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4294 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4295 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4296# endif
4297 }
4298}
4299#endif /* VBOX_STRICT */
4300
4301
4302/**
4303 * Writes a guest segment register into the guest-state area in the VMCS.
4304 *
4305 * @returns VBox status code.
4306 * @param pVCpu Pointer to the VMCPU.
4307 * @param idxSel Index of the selector in the VMCS.
4308 * @param idxLimit Index of the segment limit in the VMCS.
4309 * @param idxBase Index of the segment base in the VMCS.
4310 * @param idxAccess Index of the access rights of the segment in the VMCS.
4311 * @param pSelReg Pointer to the segment selector.
4312 *
4313 * @remarks No-long-jump zone!!!
4314 */
4315static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4316 uint32_t idxAccess, PCPUMSELREG pSelReg)
4317{
4318 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4319 AssertRCReturn(rc, rc);
4320 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4321 AssertRCReturn(rc, rc);
4322 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4323 AssertRCReturn(rc, rc);
4324
4325 uint32_t u32Access = pSelReg->Attr.u;
4326 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4327 {
4328 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4329 u32Access = 0xf3;
4330 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4331 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4332 }
4333 else
4334 {
4335 /*
4336 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4337 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4338 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4339 * loaded in protected-mode have their attribute as 0.
4340 */
4341 if (!u32Access)
4342 u32Access = X86DESCATTR_UNUSABLE;
4343 }
4344
4345 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4346 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4347 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4348
4349 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4350 AssertRCReturn(rc, rc);
4351 return rc;
4352}
4353
4354
4355/**
4356 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4357 * into the guest-state area in the VMCS.
4358 *
4359 * @returns VBox status code.
4360 * @param pVM Pointer to the VM.
4361 * @param pVCPU Pointer to the VMCPU.
4362 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4363 * out-of-sync. Make sure to update the required fields
4364 * before using them.
4365 *
4366 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4367 * @remarks No-long-jump zone!!!
4368 */
4369static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4370{
4371 int rc = VERR_INTERNAL_ERROR_5;
4372 PVM pVM = pVCpu->CTX_SUFF(pVM);
4373
4374 /*
4375 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4376 */
4377 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4378 {
4379 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4380 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4381 {
4382 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4383 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4384 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4385 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4386 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4387 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4388 }
4389
4390#ifdef VBOX_WITH_REM
4391 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4392 {
4393 Assert(pVM->hm.s.vmx.pRealModeTSS);
4394 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4395 if ( pVCpu->hm.s.vmx.fWasInRealMode
4396 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4397 {
4398 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4399 in real-mode (e.g. OpenBSD 4.0) */
4400 REMFlushTBs(pVM);
4401 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4402 pVCpu->hm.s.vmx.fWasInRealMode = false;
4403 }
4404 }
4405#endif
4406 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4407 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4408 AssertRCReturn(rc, rc);
4409 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4410 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4411 AssertRCReturn(rc, rc);
4412 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4413 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4414 AssertRCReturn(rc, rc);
4415 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4416 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4417 AssertRCReturn(rc, rc);
4418 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4419 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4420 AssertRCReturn(rc, rc);
4421 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4422 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4423 AssertRCReturn(rc, rc);
4424
4425#ifdef VBOX_STRICT
4426 /* Validate. */
4427 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4428#endif
4429
4430 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4431 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4432 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4433 }
4434
4435 /*
4436 * Guest TR.
4437 */
4438 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4439 {
4440 /*
4441 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4442 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4443 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4444 */
4445 uint16_t u16Sel = 0;
4446 uint32_t u32Limit = 0;
4447 uint64_t u64Base = 0;
4448 uint32_t u32AccessRights = 0;
4449
4450 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4451 {
4452 u16Sel = pMixedCtx->tr.Sel;
4453 u32Limit = pMixedCtx->tr.u32Limit;
4454 u64Base = pMixedCtx->tr.u64Base;
4455 u32AccessRights = pMixedCtx->tr.Attr.u;
4456 }
4457 else
4458 {
4459 Assert(pVM->hm.s.vmx.pRealModeTSS);
4460 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4461
4462 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4463 RTGCPHYS GCPhys;
4464 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4465 AssertRCReturn(rc, rc);
4466
4467 X86DESCATTR DescAttr;
4468 DescAttr.u = 0;
4469 DescAttr.n.u1Present = 1;
4470 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4471
4472 u16Sel = 0;
4473 u32Limit = HM_VTX_TSS_SIZE;
4474 u64Base = GCPhys; /* in real-mode phys = virt. */
4475 u32AccessRights = DescAttr.u;
4476 }
4477
4478 /* Validate. */
4479 Assert(!(u16Sel & RT_BIT(2)));
4480 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4481 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4482 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4483 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4484 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4485 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4486 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4487 Assert( (u32Limit & 0xfff) == 0xfff
4488 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4489 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4490 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4491
4492 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4493 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4494 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4495 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4496
4497 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4498 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4499 }
4500
4501 /*
4502 * Guest GDTR.
4503 */
4504 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4505 {
4506 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4507 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4508
4509 /* Validate. */
4510 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4511
4512 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4513 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4514 }
4515
4516 /*
4517 * Guest LDTR.
4518 */
4519 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4520 {
4521 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4522 uint32_t u32Access = 0;
4523 if (!pMixedCtx->ldtr.Attr.u)
4524 u32Access = X86DESCATTR_UNUSABLE;
4525 else
4526 u32Access = pMixedCtx->ldtr.Attr.u;
4527
4528 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4529 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4530 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4531 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4532
4533 /* Validate. */
4534 if (!(u32Access & X86DESCATTR_UNUSABLE))
4535 {
4536 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4537 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4538 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4539 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4540 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4541 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4542 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4543 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4544 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4545 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4546 }
4547
4548 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4549 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4550 }
4551
4552 /*
4553 * Guest IDTR.
4554 */
4555 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4556 {
4557 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4558 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4559
4560 /* Validate. */
4561 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4562
4563 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4564 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4565 }
4566
4567 return VINF_SUCCESS;
4568}
4569
4570
4571/**
4572 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4573 * areas.
4574 *
4575 * These MSRs will automatically be loaded to the host CPU on every successful
4576 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4577 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4578 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4579 *
4580 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4581 *
4582 * @returns VBox status code.
4583 * @param pVCpu Pointer to the VMCPU.
4584 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4585 * out-of-sync. Make sure to update the required fields
4586 * before using them.
4587 *
4588 * @remarks No-long-jump zone!!!
4589 */
4590static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4591{
4592 AssertPtr(pVCpu);
4593 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4594
4595 /*
4596 * MSRs that we use the auto-load/store MSR area in the VMCS.
4597 */
4598 PVM pVM = pVCpu->CTX_SUFF(pVM);
4599 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4600 {
4601 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4602#if HC_ARCH_BITS == 32
4603 if (pVM->hm.s.fAllow64BitGuests)
4604 {
4605 int rc = VINF_SUCCESS;
4606 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4607 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4608 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4609 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4610 AssertRCReturn(rc, rc);
4611#ifdef DEBUG
4612 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4613 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4614 {
4615 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4616 pMsr->u64Value));
4617 }
4618# endif
4619 }
4620#endif
4621 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4622 }
4623
4624 /*
4625 * Guest Sysenter MSRs.
4626 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4627 * VM-exits on WRMSRs for these MSRs.
4628 */
4629 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4630 {
4631 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4632 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4633 }
4634
4635 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4636 {
4637 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4638 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4639 }
4640
4641 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4642 {
4643 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4644 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4645 }
4646
4647 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4648 {
4649 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4650 {
4651 /*
4652 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4653 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4654 */
4655 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4656 {
4657 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4658 AssertRCReturn(rc,rc);
4659 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4660 }
4661 else
4662 {
4663 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4664 NULL /* pfAddedAndUpdated */);
4665 AssertRCReturn(rc, rc);
4666
4667 /* We need to intercept reads too, see @bugref{7386#c16}. */
4668 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4669 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4670 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4671 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4672 }
4673 }
4674 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4675 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4676 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4677 }
4678
4679 return VINF_SUCCESS;
4680}
4681
4682
4683/**
4684 * Loads the guest activity state into the guest-state area in the VMCS.
4685 *
4686 * @returns VBox status code.
4687 * @param pVCpu Pointer to the VMCPU.
4688 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4689 * out-of-sync. Make sure to update the required fields
4690 * before using them.
4691 *
4692 * @remarks No-long-jump zone!!!
4693 */
4694static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4695{
4696 NOREF(pCtx);
4697 /** @todo See if we can make use of other states, e.g.
4698 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4699 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4700 {
4701 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4702 AssertRCReturn(rc, rc);
4703
4704 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4705 }
4706 return VINF_SUCCESS;
4707}
4708
4709
4710/**
4711 * Sets up the appropriate function to run guest code.
4712 *
4713 * @returns VBox status code.
4714 * @param pVCpu Pointer to the VMCPU.
4715 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4716 * out-of-sync. Make sure to update the required fields
4717 * before using them.
4718 *
4719 * @remarks No-long-jump zone!!!
4720 */
4721static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4722{
4723 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4724 {
4725#ifndef VBOX_ENABLE_64_BITS_GUESTS
4726 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4727#endif
4728 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4729#if HC_ARCH_BITS == 32
4730 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4731 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4732 {
4733 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4734 {
4735 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4736 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4737 | HM_CHANGED_VMX_ENTRY_CTLS
4738 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4739 }
4740 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4741 }
4742#else
4743 /* 64-bit host. */
4744 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4745#endif
4746 }
4747 else
4748 {
4749 /* Guest is not in long mode, use the 32-bit handler. */
4750#if HC_ARCH_BITS == 32
4751 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4752 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4753 {
4754 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4755 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4756 | HM_CHANGED_VMX_ENTRY_CTLS
4757 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4758 }
4759#endif
4760 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4761 }
4762 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4763 return VINF_SUCCESS;
4764}
4765
4766
4767/**
4768 * Wrapper for running the guest code in VT-x.
4769 *
4770 * @returns VBox strict status code.
4771 * @param pVM Pointer to the VM.
4772 * @param pVCpu Pointer to the VMCPU.
4773 * @param pCtx Pointer to the guest-CPU context.
4774 *
4775 * @remarks No-long-jump zone!!!
4776 */
4777DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4778{
4779 /*
4780 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4781 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4782 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4783 */
4784 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4785 /** @todo Add stats for resume vs launch. */
4786#ifdef VBOX_WITH_KERNEL_USING_XMM
4787 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4788#else
4789 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4790#endif
4791}
4792
4793
4794/**
4795 * Reports world-switch error and dumps some useful debug info.
4796 *
4797 * @param pVM Pointer to the VM.
4798 * @param pVCpu Pointer to the VMCPU.
4799 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4800 * @param pCtx Pointer to the guest-CPU context.
4801 * @param pVmxTransient Pointer to the VMX transient structure (only
4802 * exitReason updated).
4803 */
4804static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4805{
4806 Assert(pVM);
4807 Assert(pVCpu);
4808 Assert(pCtx);
4809 Assert(pVmxTransient);
4810 HMVMX_ASSERT_PREEMPT_SAFE();
4811
4812 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4813 switch (rcVMRun)
4814 {
4815 case VERR_VMX_INVALID_VMXON_PTR:
4816 AssertFailed();
4817 break;
4818 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4819 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4820 {
4821 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4822 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4823 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4824 AssertRC(rc);
4825
4826 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4827 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4828 Cannot do it here as we may have been long preempted. */
4829
4830#ifdef VBOX_STRICT
4831 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4832 pVmxTransient->uExitReason));
4833 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4834 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4835 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4836 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4837 else
4838 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4839 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4840 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4841
4842 /* VMX control bits. */
4843 uint32_t u32Val;
4844 uint64_t u64Val;
4845 RTHCUINTREG uHCReg;
4846 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4847 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4848 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4849 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4850 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4851 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4852 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4853 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4854 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4855 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4856 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4857 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4858 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4859 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4860 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4861 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4862 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4863 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4864 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4865 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4866 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4867 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4868 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4869 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4870 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4871 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4872 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4873 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4874 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4875 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4876 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4877 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4878 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4879 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4880 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4881 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4882 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4883 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4884 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4885 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4886 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4887 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4888
4889 /* Guest bits. */
4890 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4891 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4892 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4893 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4894 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4895 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4896 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4897 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4898
4899 /* Host bits. */
4900 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4901 Log4(("Host CR0 %#RHr\n", uHCReg));
4902 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4903 Log4(("Host CR3 %#RHr\n", uHCReg));
4904 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4905 Log4(("Host CR4 %#RHr\n", uHCReg));
4906
4907 RTGDTR HostGdtr;
4908 PCX86DESCHC pDesc;
4909 ASMGetGDTR(&HostGdtr);
4910 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4911 Log4(("Host CS %#08x\n", u32Val));
4912 if (u32Val < HostGdtr.cbGdt)
4913 {
4914 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4915 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4916 }
4917
4918 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4919 Log4(("Host DS %#08x\n", u32Val));
4920 if (u32Val < HostGdtr.cbGdt)
4921 {
4922 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4923 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4924 }
4925
4926 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4927 Log4(("Host ES %#08x\n", u32Val));
4928 if (u32Val < HostGdtr.cbGdt)
4929 {
4930 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4931 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4932 }
4933
4934 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4935 Log4(("Host FS %#08x\n", u32Val));
4936 if (u32Val < HostGdtr.cbGdt)
4937 {
4938 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4939 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4940 }
4941
4942 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4943 Log4(("Host GS %#08x\n", u32Val));
4944 if (u32Val < HostGdtr.cbGdt)
4945 {
4946 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4947 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4948 }
4949
4950 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4951 Log4(("Host SS %#08x\n", u32Val));
4952 if (u32Val < HostGdtr.cbGdt)
4953 {
4954 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4955 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4956 }
4957
4958 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4959 Log4(("Host TR %#08x\n", u32Val));
4960 if (u32Val < HostGdtr.cbGdt)
4961 {
4962 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4963 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4964 }
4965
4966 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4967 Log4(("Host TR Base %#RHv\n", uHCReg));
4968 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4969 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4970 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4971 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4972 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4973 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4974 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4975 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4976 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4977 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4978 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4979 Log4(("Host RSP %#RHv\n", uHCReg));
4980 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4981 Log4(("Host RIP %#RHv\n", uHCReg));
4982# if HC_ARCH_BITS == 64
4983 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4984 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4985 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4986 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4987 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4988 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4989# endif
4990#endif /* VBOX_STRICT */
4991 break;
4992 }
4993
4994 default:
4995 /* Impossible */
4996 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4997 break;
4998 }
4999 NOREF(pVM); NOREF(pCtx);
5000}
5001
5002
5003#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5004#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5005# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5006#endif
5007#ifdef VBOX_STRICT
5008static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5009{
5010 switch (idxField)
5011 {
5012 case VMX_VMCS_GUEST_RIP:
5013 case VMX_VMCS_GUEST_RSP:
5014 case VMX_VMCS_GUEST_SYSENTER_EIP:
5015 case VMX_VMCS_GUEST_SYSENTER_ESP:
5016 case VMX_VMCS_GUEST_GDTR_BASE:
5017 case VMX_VMCS_GUEST_IDTR_BASE:
5018 case VMX_VMCS_GUEST_CS_BASE:
5019 case VMX_VMCS_GUEST_DS_BASE:
5020 case VMX_VMCS_GUEST_ES_BASE:
5021 case VMX_VMCS_GUEST_FS_BASE:
5022 case VMX_VMCS_GUEST_GS_BASE:
5023 case VMX_VMCS_GUEST_SS_BASE:
5024 case VMX_VMCS_GUEST_LDTR_BASE:
5025 case VMX_VMCS_GUEST_TR_BASE:
5026 case VMX_VMCS_GUEST_CR3:
5027 return true;
5028 }
5029 return false;
5030}
5031
5032static bool hmR0VmxIsValidReadField(uint32_t idxField)
5033{
5034 switch (idxField)
5035 {
5036 /* Read-only fields. */
5037 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5038 return true;
5039 }
5040 /* Remaining readable fields should also be writable. */
5041 return hmR0VmxIsValidWriteField(idxField);
5042}
5043#endif /* VBOX_STRICT */
5044
5045
5046/**
5047 * Executes the specified handler in 64-bit mode.
5048 *
5049 * @returns VBox status code.
5050 * @param pVM Pointer to the VM.
5051 * @param pVCpu Pointer to the VMCPU.
5052 * @param pCtx Pointer to the guest CPU context.
5053 * @param enmOp The operation to perform.
5054 * @param cParams Number of parameters.
5055 * @param paParam Array of 32-bit parameters.
5056 */
5057VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5058 uint32_t cParams, uint32_t *paParam)
5059{
5060 int rc, rc2;
5061 PHMGLOBALCPUINFO pCpu;
5062 RTHCPHYS HCPhysCpuPage;
5063 RTCCUINTREG fOldEFlags;
5064
5065 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5066 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5067 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5068 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5069
5070#ifdef VBOX_STRICT
5071 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5072 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5073
5074 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5075 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5076#endif
5077
5078 /* Disable interrupts. */
5079 fOldEFlags = ASMIntDisableFlags();
5080
5081#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5082 RTCPUID idHostCpu = RTMpCpuId();
5083 CPUMR0SetLApic(pVCpu, idHostCpu);
5084#endif
5085
5086 pCpu = HMR0GetCurrentCpu();
5087 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5088
5089 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5090 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5091
5092 /* Leave VMX Root Mode. */
5093 VMXDisable();
5094
5095 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5096
5097 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5098 CPUMSetHyperEIP(pVCpu, enmOp);
5099 for (int i = (int)cParams - 1; i >= 0; i--)
5100 CPUMPushHyper(pVCpu, paParam[i]);
5101
5102 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5103
5104 /* Call the switcher. */
5105 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5106 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5107
5108 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5109 /* Make sure the VMX instructions don't cause #UD faults. */
5110 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5111
5112 /* Re-enter VMX Root Mode */
5113 rc2 = VMXEnable(HCPhysCpuPage);
5114 if (RT_FAILURE(rc2))
5115 {
5116 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5117 ASMSetFlags(fOldEFlags);
5118 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5119 return rc2;
5120 }
5121
5122 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5123 AssertRC(rc2);
5124 Assert(!(ASMGetFlags() & X86_EFL_IF));
5125 ASMSetFlags(fOldEFlags);
5126 return rc;
5127}
5128
5129
5130/**
5131 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5132 * supporting 64-bit guests.
5133 *
5134 * @returns VBox status code.
5135 * @param fResume Whether to VMLAUNCH or VMRESUME.
5136 * @param pCtx Pointer to the guest-CPU context.
5137 * @param pCache Pointer to the VMCS cache.
5138 * @param pVM Pointer to the VM.
5139 * @param pVCpu Pointer to the VMCPU.
5140 */
5141DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5142{
5143 PHMGLOBALCPUINFO pCpu = NULL;
5144 RTHCPHYS HCPhysCpuPage = 0;
5145 int rc = VERR_INTERNAL_ERROR_5;
5146
5147 pCpu = HMR0GetCurrentCpu();
5148 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5149
5150#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5151 pCache->uPos = 1;
5152 pCache->interPD = PGMGetInterPaeCR3(pVM);
5153 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5154#endif
5155
5156#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5157 pCache->TestIn.HCPhysCpuPage = 0;
5158 pCache->TestIn.HCPhysVmcs = 0;
5159 pCache->TestIn.pCache = 0;
5160 pCache->TestOut.HCPhysVmcs = 0;
5161 pCache->TestOut.pCache = 0;
5162 pCache->TestOut.pCtx = 0;
5163 pCache->TestOut.eflags = 0;
5164#endif
5165
5166 uint32_t aParam[10];
5167 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5168 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5169 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5170 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5171 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5172 aParam[5] = 0;
5173 aParam[6] = VM_RC_ADDR(pVM, pVM);
5174 aParam[7] = 0;
5175 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5176 aParam[9] = 0;
5177
5178#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5179 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5180 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5181#endif
5182 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5183
5184#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5185 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5186 Assert(pCtx->dr[4] == 10);
5187 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5188#endif
5189
5190#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5191 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5192 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5193 pVCpu->hm.s.vmx.HCPhysVmcs));
5194 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5195 pCache->TestOut.HCPhysVmcs));
5196 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5197 pCache->TestOut.pCache));
5198 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5199 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5200 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5201 pCache->TestOut.pCtx));
5202 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5203#endif
5204 return rc;
5205}
5206
5207
5208/**
5209 * Initialize the VMCS-Read cache.
5210 *
5211 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5212 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5213 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5214 * (those that have a 32-bit FULL & HIGH part).
5215 *
5216 * @returns VBox status code.
5217 * @param pVM Pointer to the VM.
5218 * @param pVCpu Pointer to the VMCPU.
5219 */
5220static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5221{
5222#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5223{ \
5224 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5225 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5226 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5227 ++cReadFields; \
5228}
5229
5230 AssertPtr(pVM);
5231 AssertPtr(pVCpu);
5232 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5233 uint32_t cReadFields = 0;
5234
5235 /*
5236 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5237 * and serve to indicate exceptions to the rules.
5238 */
5239
5240 /* Guest-natural selector base fields. */
5241#if 0
5242 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5243 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5244 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5245#endif
5246 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5247 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5248 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5249 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5250 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5251 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5252 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5253 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5254 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5255 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5256 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5257 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5258#if 0
5259 /* Unused natural width guest-state fields. */
5260 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5261 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5262#endif
5263 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5264 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5265
5266 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5267#if 0
5268 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5269 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5270 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5271 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5272 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5273 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5274 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5275 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5276 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5277#endif
5278
5279 /* Natural width guest-state fields. */
5280 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5281#if 0
5282 /* Currently unused field. */
5283 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5284#endif
5285
5286 if (pVM->hm.s.fNestedPaging)
5287 {
5288 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5289 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5290 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5291 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5292 }
5293 else
5294 {
5295 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5296 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5297 }
5298
5299#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5300 return VINF_SUCCESS;
5301}
5302
5303
5304/**
5305 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5306 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5307 * darwin, running 64-bit guests).
5308 *
5309 * @returns VBox status code.
5310 * @param pVCpu Pointer to the VMCPU.
5311 * @param idxField The VMCS field encoding.
5312 * @param u64Val 16, 32 or 64-bit value.
5313 */
5314VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5315{
5316 int rc;
5317 switch (idxField)
5318 {
5319 /*
5320 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5321 */
5322 /* 64-bit Control fields. */
5323 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5324 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5325 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5326 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5327 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5328 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5329 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5330 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5331 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5332 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5333 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5334 case VMX_VMCS64_CTRL_EPTP_FULL:
5335 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5336 /* 64-bit Guest-state fields. */
5337 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5338 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5339 case VMX_VMCS64_GUEST_PAT_FULL:
5340 case VMX_VMCS64_GUEST_EFER_FULL:
5341 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5342 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5343 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5344 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5345 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5346 /* 64-bit Host-state fields. */
5347 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5348 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5349 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5350 {
5351 rc = VMXWriteVmcs32(idxField, u64Val);
5352 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5353 break;
5354 }
5355
5356 /*
5357 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5358 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5359 */
5360 /* Natural-width Guest-state fields. */
5361 case VMX_VMCS_GUEST_CR3:
5362 case VMX_VMCS_GUEST_ES_BASE:
5363 case VMX_VMCS_GUEST_CS_BASE:
5364 case VMX_VMCS_GUEST_SS_BASE:
5365 case VMX_VMCS_GUEST_DS_BASE:
5366 case VMX_VMCS_GUEST_FS_BASE:
5367 case VMX_VMCS_GUEST_GS_BASE:
5368 case VMX_VMCS_GUEST_LDTR_BASE:
5369 case VMX_VMCS_GUEST_TR_BASE:
5370 case VMX_VMCS_GUEST_GDTR_BASE:
5371 case VMX_VMCS_GUEST_IDTR_BASE:
5372 case VMX_VMCS_GUEST_RSP:
5373 case VMX_VMCS_GUEST_RIP:
5374 case VMX_VMCS_GUEST_SYSENTER_ESP:
5375 case VMX_VMCS_GUEST_SYSENTER_EIP:
5376 {
5377 if (!(u64Val >> 32))
5378 {
5379 /* If this field is 64-bit, VT-x will zero out the top bits. */
5380 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5381 }
5382 else
5383 {
5384 /* Assert that only the 32->64 switcher case should ever come here. */
5385 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5386 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5387 }
5388 break;
5389 }
5390
5391 default:
5392 {
5393 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5394 rc = VERR_INVALID_PARAMETER;
5395 break;
5396 }
5397 }
5398 AssertRCReturn(rc, rc);
5399 return rc;
5400}
5401
5402
5403/**
5404 * Queue up a VMWRITE by using the VMCS write cache.
5405 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5406 *
5407 * @param pVCpu Pointer to the VMCPU.
5408 * @param idxField The VMCS field encoding.
5409 * @param u64Val 16, 32 or 64-bit value.
5410 */
5411VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5412{
5413 AssertPtr(pVCpu);
5414 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5415
5416 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5417 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5418
5419 /* Make sure there are no duplicates. */
5420 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5421 {
5422 if (pCache->Write.aField[i] == idxField)
5423 {
5424 pCache->Write.aFieldVal[i] = u64Val;
5425 return VINF_SUCCESS;
5426 }
5427 }
5428
5429 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5430 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5431 pCache->Write.cValidEntries++;
5432 return VINF_SUCCESS;
5433}
5434#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5435
5436
5437/**
5438 * Sets up the usage of TSC-offsetting and updates the VMCS.
5439 *
5440 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5441 * VMX preemption timer.
5442 *
5443 * @returns VBox status code.
5444 * @param pVM Pointer to the cross context VM structure.
5445 * @param pVCpu Pointer to the VMCPU.
5446 *
5447 * @remarks No-long-jump zone!!!
5448 */
5449static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5450{
5451 int rc;
5452 bool fOffsettedTsc;
5453 bool fParavirtTsc;
5454 if (pVM->hm.s.vmx.fUsePreemptTimer)
5455 {
5456 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5457 &fOffsettedTsc, &fParavirtTsc);
5458
5459 /* Make sure the returned values have sane upper and lower boundaries. */
5460 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5461 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5462 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5463 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5464
5465 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5466 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5467 }
5468 else
5469 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5470
5471 /** @todo later optimize this to be done elsewhere and not before every
5472 * VM-entry. */
5473 if (fParavirtTsc)
5474 {
5475 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5476 information before every VM-entry, hence disable it for performance sake. */
5477#if 0
5478 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5479 AssertRC(rc);
5480#endif
5481 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5482 }
5483
5484 if (fOffsettedTsc)
5485 {
5486 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5487 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5488
5489 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5490 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5491 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5492 }
5493 else
5494 {
5495 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5496 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5497 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5498 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5499 }
5500}
5501
5502
5503/**
5504 * Determines if an exception is a contributory exception.
5505 *
5506 * Contributory exceptions are ones which can cause double-faults unless the
5507 * original exception was a benign exception. Page-fault is intentionally not
5508 * included here as it's a conditional contributory exception.
5509 *
5510 * @returns true if the exception is contributory, false otherwise.
5511 * @param uVector The exception vector.
5512 */
5513DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5514{
5515 switch (uVector)
5516 {
5517 case X86_XCPT_GP:
5518 case X86_XCPT_SS:
5519 case X86_XCPT_NP:
5520 case X86_XCPT_TS:
5521 case X86_XCPT_DE:
5522 return true;
5523 default:
5524 break;
5525 }
5526 return false;
5527}
5528
5529
5530/**
5531 * Sets an event as a pending event to be injected into the guest.
5532 *
5533 * @param pVCpu Pointer to the VMCPU.
5534 * @param u32IntInfo The VM-entry interruption-information field.
5535 * @param cbInstr The VM-entry instruction length in bytes (for software
5536 * interrupts, exceptions and privileged software
5537 * exceptions).
5538 * @param u32ErrCode The VM-entry exception error code.
5539 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5540 * page-fault.
5541 *
5542 * @remarks Statistics counter assumes this is a guest event being injected or
5543 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5544 * always incremented.
5545 */
5546DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5547 RTGCUINTPTR GCPtrFaultAddress)
5548{
5549 Assert(!pVCpu->hm.s.Event.fPending);
5550 pVCpu->hm.s.Event.fPending = true;
5551 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5552 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5553 pVCpu->hm.s.Event.cbInstr = cbInstr;
5554 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5555
5556 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5557}
5558
5559
5560/**
5561 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5562 *
5563 * @param pVCpu Pointer to the VMCPU.
5564 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5565 * out-of-sync. Make sure to update the required fields
5566 * before using them.
5567 */
5568DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5569{
5570 NOREF(pMixedCtx);
5571 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5572 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5573 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5574 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5575}
5576
5577
5578/**
5579 * Handle a condition that occurred while delivering an event through the guest
5580 * IDT.
5581 *
5582 * @returns VBox status code (informational error codes included).
5583 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5584 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5585 * continue execution of the guest which will delivery the #DF.
5586 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5587 *
5588 * @param pVCpu Pointer to the VMCPU.
5589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5590 * out-of-sync. Make sure to update the required fields
5591 * before using them.
5592 * @param pVmxTransient Pointer to the VMX transient structure.
5593 *
5594 * @remarks No-long-jump zone!!!
5595 */
5596static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5597{
5598 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5599
5600 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5601 AssertRCReturn(rc, rc);
5602 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5603 AssertRCReturn(rc, rc);
5604
5605 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5606 {
5607 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5608 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5609
5610 typedef enum
5611 {
5612 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5613 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5614 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5615 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5616 } VMXREFLECTXCPT;
5617
5618 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5619 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5620 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5621 {
5622 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5623 {
5624 enmReflect = VMXREFLECTXCPT_XCPT;
5625#ifdef VBOX_STRICT
5626 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5627 && uExitVector == X86_XCPT_PF)
5628 {
5629 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5630 }
5631#endif
5632 if ( uExitVector == X86_XCPT_PF
5633 && uIdtVector == X86_XCPT_PF)
5634 {
5635 pVmxTransient->fVectoringDoublePF = true;
5636 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5637 }
5638 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5639 && hmR0VmxIsContributoryXcpt(uExitVector)
5640 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5641 || uIdtVector == X86_XCPT_PF))
5642 {
5643 enmReflect = VMXREFLECTXCPT_DF;
5644 }
5645 else if (uIdtVector == X86_XCPT_DF)
5646 enmReflect = VMXREFLECTXCPT_TF;
5647 }
5648 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5649 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5650 {
5651 /*
5652 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5653 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5654 */
5655 enmReflect = VMXREFLECTXCPT_XCPT;
5656
5657 if (uExitVector == X86_XCPT_PF)
5658 {
5659 pVmxTransient->fVectoringPF = true;
5660 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5661 }
5662 }
5663 }
5664 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5665 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5666 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5667 {
5668 /*
5669 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5670 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5671 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5672 */
5673 enmReflect = VMXREFLECTXCPT_XCPT;
5674 }
5675
5676 /*
5677 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5678 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5679 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5680 *
5681 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5682 */
5683 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5684 && enmReflect == VMXREFLECTXCPT_XCPT
5685 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5686 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5687 {
5688 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5689 }
5690
5691 switch (enmReflect)
5692 {
5693 case VMXREFLECTXCPT_XCPT:
5694 {
5695 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5696 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5697 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5698
5699 uint32_t u32ErrCode = 0;
5700 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5701 {
5702 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5703 AssertRCReturn(rc, rc);
5704 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5705 }
5706
5707 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5708 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5709 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5710 rc = VINF_SUCCESS;
5711 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5712 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5713
5714 break;
5715 }
5716
5717 case VMXREFLECTXCPT_DF:
5718 {
5719 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5720 rc = VINF_HM_DOUBLE_FAULT;
5721 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5722 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5723
5724 break;
5725 }
5726
5727 case VMXREFLECTXCPT_TF:
5728 {
5729 rc = VINF_EM_RESET;
5730 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5731 uExitVector));
5732 break;
5733 }
5734
5735 default:
5736 Assert(rc == VINF_SUCCESS);
5737 break;
5738 }
5739 }
5740 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5741 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5742 && uExitVector != X86_XCPT_DF
5743 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5744 {
5745 /*
5746 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5747 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5748 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5749 */
5750 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5751 {
5752 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5753 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5754 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5755 }
5756 }
5757
5758 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5759 return rc;
5760}
5761
5762
5763/**
5764 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5765 *
5766 * @returns VBox status code.
5767 * @param pVCpu Pointer to the VMCPU.
5768 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5769 * out-of-sync. Make sure to update the required fields
5770 * before using them.
5771 *
5772 * @remarks No-long-jump zone!!!
5773 */
5774static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5775{
5776 NOREF(pMixedCtx);
5777
5778 /*
5779 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5780 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5781 */
5782 VMMRZCallRing3Disable(pVCpu);
5783 HM_DISABLE_PREEMPT();
5784
5785 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5786 {
5787 uint32_t uVal = 0;
5788 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5789 AssertRCReturn(rc, rc);
5790
5791 uint32_t uShadow = 0;
5792 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5793 AssertRCReturn(rc, rc);
5794
5795 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5796 CPUMSetGuestCR0(pVCpu, uVal);
5797 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5798 }
5799
5800 HM_RESTORE_PREEMPT();
5801 VMMRZCallRing3Enable(pVCpu);
5802 return VINF_SUCCESS;
5803}
5804
5805
5806/**
5807 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5808 *
5809 * @returns VBox status code.
5810 * @param pVCpu Pointer to the VMCPU.
5811 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5812 * out-of-sync. Make sure to update the required fields
5813 * before using them.
5814 *
5815 * @remarks No-long-jump zone!!!
5816 */
5817static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5818{
5819 NOREF(pMixedCtx);
5820
5821 int rc = VINF_SUCCESS;
5822 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5823 {
5824 uint32_t uVal = 0;
5825 uint32_t uShadow = 0;
5826 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5827 AssertRCReturn(rc, rc);
5828 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5829 AssertRCReturn(rc, rc);
5830
5831 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5832 CPUMSetGuestCR4(pVCpu, uVal);
5833 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5834 }
5835 return rc;
5836}
5837
5838
5839/**
5840 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5841 *
5842 * @returns VBox status code.
5843 * @param pVCpu Pointer to the VMCPU.
5844 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5845 * out-of-sync. Make sure to update the required fields
5846 * before using them.
5847 *
5848 * @remarks No-long-jump zone!!!
5849 */
5850static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5851{
5852 int rc = VINF_SUCCESS;
5853 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5854 {
5855 uint64_t u64Val = 0;
5856 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5857 AssertRCReturn(rc, rc);
5858
5859 pMixedCtx->rip = u64Val;
5860 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5861 }
5862 return rc;
5863}
5864
5865
5866/**
5867 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5868 *
5869 * @returns VBox status code.
5870 * @param pVCpu Pointer to the VMCPU.
5871 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5872 * out-of-sync. Make sure to update the required fields
5873 * before using them.
5874 *
5875 * @remarks No-long-jump zone!!!
5876 */
5877static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5878{
5879 int rc = VINF_SUCCESS;
5880 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5881 {
5882 uint64_t u64Val = 0;
5883 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5884 AssertRCReturn(rc, rc);
5885
5886 pMixedCtx->rsp = u64Val;
5887 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5888 }
5889 return rc;
5890}
5891
5892
5893/**
5894 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5895 *
5896 * @returns VBox status code.
5897 * @param pVCpu Pointer to the VMCPU.
5898 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5899 * out-of-sync. Make sure to update the required fields
5900 * before using them.
5901 *
5902 * @remarks No-long-jump zone!!!
5903 */
5904static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5905{
5906 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5907 {
5908 uint32_t uVal = 0;
5909 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5910 AssertRCReturn(rc, rc);
5911
5912 pMixedCtx->eflags.u32 = uVal;
5913 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5914 {
5915 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5916 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5917
5918 pMixedCtx->eflags.Bits.u1VM = 0;
5919 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5920 }
5921
5922 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5923 }
5924 return VINF_SUCCESS;
5925}
5926
5927
5928/**
5929 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5930 * guest-CPU context.
5931 */
5932DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5933{
5934 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5935 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5936 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5937 return rc;
5938}
5939
5940
5941/**
5942 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5943 * from the guest-state area in the VMCS.
5944 *
5945 * @param pVCpu Pointer to the VMCPU.
5946 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5947 * out-of-sync. Make sure to update the required fields
5948 * before using them.
5949 *
5950 * @remarks No-long-jump zone!!!
5951 */
5952static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5953{
5954 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
5955 {
5956 uint32_t uIntrState = 0;
5957 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5958 AssertRC(rc);
5959
5960 if (!uIntrState)
5961 {
5962 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5963 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5964
5965 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5966 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5967 }
5968 else
5969 {
5970 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
5971 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
5972 {
5973 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5974 AssertRC(rc);
5975 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5976 AssertRC(rc);
5977
5978 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5979 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5980 }
5981 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5982 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5983
5984 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
5985 {
5986 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5987 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5988 }
5989 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5990 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5991 }
5992
5993 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
5994 }
5995}
5996
5997
5998/**
5999 * Saves the guest's activity state.
6000 *
6001 * @returns VBox status code.
6002 * @param pVCpu Pointer to the VMCPU.
6003 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6004 * out-of-sync. Make sure to update the required fields
6005 * before using them.
6006 *
6007 * @remarks No-long-jump zone!!!
6008 */
6009static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6010{
6011 NOREF(pMixedCtx);
6012 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6013 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6014 return VINF_SUCCESS;
6015}
6016
6017
6018/**
6019 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6020 * the current VMCS into the guest-CPU context.
6021 *
6022 * @returns VBox status code.
6023 * @param pVCpu Pointer to the VMCPU.
6024 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6025 * out-of-sync. Make sure to update the required fields
6026 * before using them.
6027 *
6028 * @remarks No-long-jump zone!!!
6029 */
6030static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6031{
6032 int rc = VINF_SUCCESS;
6033 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6034 {
6035 uint32_t u32Val = 0;
6036 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6037 pMixedCtx->SysEnter.cs = u32Val;
6038 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6039 }
6040
6041 uint64_t u64Val = 0;
6042 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6043 {
6044 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6045 pMixedCtx->SysEnter.eip = u64Val;
6046 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6047 }
6048 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6049 {
6050 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6051 pMixedCtx->SysEnter.esp = u64Val;
6052 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6053 }
6054 return rc;
6055}
6056
6057
6058/**
6059 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6060 * the CPU back into the guest-CPU context.
6061 *
6062 * @returns VBox status code.
6063 * @param pVCpu Pointer to the VMCPU.
6064 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6065 * out-of-sync. Make sure to update the required fields
6066 * before using them.
6067 *
6068 * @remarks No-long-jump zone!!!
6069 */
6070static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6071{
6072#if HC_ARCH_BITS == 64
6073 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6074 {
6075 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6076 VMMRZCallRing3Disable(pVCpu);
6077 HM_DISABLE_PREEMPT();
6078
6079 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6080 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6081 {
6082 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6083 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6084 }
6085
6086 HM_RESTORE_PREEMPT();
6087 VMMRZCallRing3Enable(pVCpu);
6088 }
6089 else
6090 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6091#else
6092 NOREF(pMixedCtx);
6093 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6094#endif
6095
6096 return VINF_SUCCESS;
6097}
6098
6099
6100/**
6101 * Saves the auto load/store'd guest MSRs from the current VMCS into
6102 * the guest-CPU context.
6103 *
6104 * @returns VBox status code.
6105 * @param pVCpu Pointer to the VMCPU.
6106 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6107 * out-of-sync. Make sure to update the required fields
6108 * before using them.
6109 *
6110 * @remarks No-long-jump zone!!!
6111 */
6112static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6113{
6114 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6115 return VINF_SUCCESS;
6116
6117 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6118 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6119 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6120 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6121 {
6122 switch (pMsr->u32Msr)
6123 {
6124 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6125 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6126 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6127 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6128 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6129 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6130 break;
6131
6132 default:
6133 {
6134 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6135 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6136 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6137 }
6138 }
6139 }
6140
6141 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6142 return VINF_SUCCESS;
6143}
6144
6145
6146/**
6147 * Saves the guest control registers from the current VMCS into the guest-CPU
6148 * context.
6149 *
6150 * @returns VBox status code.
6151 * @param pVCpu Pointer to the VMCPU.
6152 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6153 * out-of-sync. Make sure to update the required fields
6154 * before using them.
6155 *
6156 * @remarks No-long-jump zone!!!
6157 */
6158static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6159{
6160 /* Guest CR0. Guest FPU. */
6161 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6162 AssertRCReturn(rc, rc);
6163
6164 /* Guest CR4. */
6165 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6166 AssertRCReturn(rc, rc);
6167
6168 /* Guest CR2 - updated always during the world-switch or in #PF. */
6169 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6170 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6171 {
6172 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6173 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6174
6175 PVM pVM = pVCpu->CTX_SUFF(pVM);
6176 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6177 || ( pVM->hm.s.fNestedPaging
6178 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6179 {
6180 uint64_t u64Val = 0;
6181 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6182 if (pMixedCtx->cr3 != u64Val)
6183 {
6184 CPUMSetGuestCR3(pVCpu, u64Val);
6185 if (VMMRZCallRing3IsEnabled(pVCpu))
6186 {
6187 PGMUpdateCR3(pVCpu, u64Val);
6188 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6189 }
6190 else
6191 {
6192 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6193 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6194 }
6195 }
6196
6197 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6198 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6199 {
6200 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6201 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6202 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6203 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6204
6205 if (VMMRZCallRing3IsEnabled(pVCpu))
6206 {
6207 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6208 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6209 }
6210 else
6211 {
6212 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6213 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6214 }
6215 }
6216 }
6217
6218 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6219 }
6220
6221 /*
6222 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6223 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6224 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6225 *
6226 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6227 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6228 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6229 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6230 *
6231 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6232 */
6233 if (VMMRZCallRing3IsEnabled(pVCpu))
6234 {
6235 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6236 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6237
6238 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6239 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6240
6241 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6242 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6243 }
6244
6245 return rc;
6246}
6247
6248
6249/**
6250 * Reads a guest segment register from the current VMCS into the guest-CPU
6251 * context.
6252 *
6253 * @returns VBox status code.
6254 * @param pVCpu Pointer to the VMCPU.
6255 * @param idxSel Index of the selector in the VMCS.
6256 * @param idxLimit Index of the segment limit in the VMCS.
6257 * @param idxBase Index of the segment base in the VMCS.
6258 * @param idxAccess Index of the access rights of the segment in the VMCS.
6259 * @param pSelReg Pointer to the segment selector.
6260 *
6261 * @remarks No-long-jump zone!!!
6262 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6263 * macro as that takes care of whether to read from the VMCS cache or
6264 * not.
6265 */
6266DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6267 PCPUMSELREG pSelReg)
6268{
6269 NOREF(pVCpu);
6270
6271 uint32_t u32Val = 0;
6272 int rc = VMXReadVmcs32(idxSel, &u32Val);
6273 AssertRCReturn(rc, rc);
6274 pSelReg->Sel = (uint16_t)u32Val;
6275 pSelReg->ValidSel = (uint16_t)u32Val;
6276 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6277
6278 rc = VMXReadVmcs32(idxLimit, &u32Val);
6279 AssertRCReturn(rc, rc);
6280 pSelReg->u32Limit = u32Val;
6281
6282 uint64_t u64Val = 0;
6283 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6284 AssertRCReturn(rc, rc);
6285 pSelReg->u64Base = u64Val;
6286
6287 rc = VMXReadVmcs32(idxAccess, &u32Val);
6288 AssertRCReturn(rc, rc);
6289 pSelReg->Attr.u = u32Val;
6290
6291 /*
6292 * If VT-x marks the segment as unusable, most other bits remain undefined:
6293 * - For CS the L, D and G bits have meaning.
6294 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6295 * - For the remaining data segments no bits are defined.
6296 *
6297 * The present bit and the unusable bit has been observed to be set at the
6298 * same time (the selector was supposed to be invalid as we started executing
6299 * a V8086 interrupt in ring-0).
6300 *
6301 * What should be important for the rest of the VBox code, is that the P bit is
6302 * cleared. Some of the other VBox code recognizes the unusable bit, but
6303 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6304 * safe side here, we'll strip off P and other bits we don't care about. If
6305 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6306 *
6307 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6308 */
6309 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6310 {
6311 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6312
6313 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6314 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6315 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6316
6317 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6318#ifdef DEBUG_bird
6319 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6320 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6321 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6322#endif
6323 }
6324 return VINF_SUCCESS;
6325}
6326
6327
6328#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6329# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6330 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6331 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6332#else
6333# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6334 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6335 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6336#endif
6337
6338
6339/**
6340 * Saves the guest segment registers from the current VMCS into the guest-CPU
6341 * context.
6342 *
6343 * @returns VBox status code.
6344 * @param pVCpu Pointer to the VMCPU.
6345 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6346 * out-of-sync. Make sure to update the required fields
6347 * before using them.
6348 *
6349 * @remarks No-long-jump zone!!!
6350 */
6351static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6352{
6353 /* Guest segment registers. */
6354 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6355 {
6356 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6357 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6358 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6359 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6360 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6361 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6362 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6363
6364 /* Restore segment attributes for real-on-v86 mode hack. */
6365 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6366 {
6367 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6368 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6369 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6370 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6371 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6372 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6373 }
6374 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6375 }
6376
6377 return VINF_SUCCESS;
6378}
6379
6380
6381/**
6382 * Saves the guest descriptor table registers and task register from the current
6383 * VMCS into the guest-CPU context.
6384 *
6385 * @returns VBox status code.
6386 * @param pVCpu Pointer to the VMCPU.
6387 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6388 * out-of-sync. Make sure to update the required fields
6389 * before using them.
6390 *
6391 * @remarks No-long-jump zone!!!
6392 */
6393static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6394{
6395 int rc = VINF_SUCCESS;
6396
6397 /* Guest LDTR. */
6398 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6399 {
6400 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6401 AssertRCReturn(rc, rc);
6402 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6403 }
6404
6405 /* Guest GDTR. */
6406 uint64_t u64Val = 0;
6407 uint32_t u32Val = 0;
6408 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6409 {
6410 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6411 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6412 pMixedCtx->gdtr.pGdt = u64Val;
6413 pMixedCtx->gdtr.cbGdt = u32Val;
6414 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6415 }
6416
6417 /* Guest IDTR. */
6418 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6419 {
6420 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6421 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6422 pMixedCtx->idtr.pIdt = u64Val;
6423 pMixedCtx->idtr.cbIdt = u32Val;
6424 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6425 }
6426
6427 /* Guest TR. */
6428 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6429 {
6430 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6431 AssertRCReturn(rc, rc);
6432
6433 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6434 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6435 {
6436 rc = VMXLOCAL_READ_SEG(TR, tr);
6437 AssertRCReturn(rc, rc);
6438 }
6439 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6440 }
6441 return rc;
6442}
6443
6444#undef VMXLOCAL_READ_SEG
6445
6446
6447/**
6448 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6449 * context.
6450 *
6451 * @returns VBox status code.
6452 * @param pVCpu Pointer to the VMCPU.
6453 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6454 * out-of-sync. Make sure to update the required fields
6455 * before using them.
6456 *
6457 * @remarks No-long-jump zone!!!
6458 */
6459static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6460{
6461 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6462 {
6463 if (!pVCpu->hm.s.fUsingHyperDR7)
6464 {
6465 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6466 uint32_t u32Val;
6467 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6468 pMixedCtx->dr[7] = u32Val;
6469 }
6470
6471 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6472 }
6473 return VINF_SUCCESS;
6474}
6475
6476
6477/**
6478 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6479 *
6480 * @returns VBox status code.
6481 * @param pVCpu Pointer to the VMCPU.
6482 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6483 * out-of-sync. Make sure to update the required fields
6484 * before using them.
6485 *
6486 * @remarks No-long-jump zone!!!
6487 */
6488static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6489{
6490 NOREF(pMixedCtx);
6491
6492 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6493 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6494 return VINF_SUCCESS;
6495}
6496
6497
6498/**
6499 * Saves the entire guest state from the currently active VMCS into the
6500 * guest-CPU context.
6501 *
6502 * This essentially VMREADs all guest-data.
6503 *
6504 * @returns VBox status code.
6505 * @param pVCpu Pointer to the VMCPU.
6506 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6507 * out-of-sync. Make sure to update the required fields
6508 * before using them.
6509 */
6510static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6511{
6512 Assert(pVCpu);
6513 Assert(pMixedCtx);
6514
6515 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6516 return VINF_SUCCESS;
6517
6518 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6519 again on the ring-3 callback path, there is no real need to. */
6520 if (VMMRZCallRing3IsEnabled(pVCpu))
6521 VMMR0LogFlushDisable(pVCpu);
6522 else
6523 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6524 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6525
6526 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6527 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6528
6529 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6530 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6531
6532 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6533 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6534
6535 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6536 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6537
6538 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6539 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6540
6541 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6542 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6543
6544 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6545 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6546
6547 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6548 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6549
6550 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6551 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6552
6553 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6554 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6555
6556 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6557 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6558
6559 if (VMMRZCallRing3IsEnabled(pVCpu))
6560 VMMR0LogFlushEnable(pVCpu);
6561
6562 return VINF_SUCCESS;
6563}
6564
6565
6566/**
6567 * Saves basic guest registers needed for IEM instruction execution.
6568 *
6569 * @returns VBox status code (OR-able).
6570 * @param pVCpu Pointer to the cross context CPU data for the calling
6571 * EMT.
6572 * @param pMixedCtx Pointer to the CPU context of the guest.
6573 * @param fMemory Whether the instruction being executed operates on
6574 * memory or not. Only CR0 is synced up if clear.
6575 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6576 */
6577static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6578{
6579 /*
6580 * We assume all general purpose registers other than RSP are available.
6581 *
6582 * RIP is a must, as it will be incremented or otherwise changed.
6583 *
6584 * RFLAGS are always required to figure the CPL.
6585 *
6586 * RSP isn't always required, however it's a GPR, so frequently required.
6587 *
6588 * SS and CS are the only segment register needed if IEM doesn't do memory
6589 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6590 *
6591 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6592 * be required for memory accesses.
6593 *
6594 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6595 */
6596 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6597 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6598 if (fNeedRsp)
6599 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6600 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6601 if (!fMemory)
6602 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6603 else
6604 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6605 return rc;
6606}
6607
6608
6609/**
6610 * Ensures that we've got a complete basic guest-context.
6611 *
6612 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6613 * is for the interpreter.
6614 *
6615 * @returns VBox status code.
6616 * @param pVCpu Pointer to the VMCPU of the calling EMT.
6617 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6618 * needing to be synced in.
6619 * @thread EMT(pVCpu)
6620 */
6621VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6622{
6623 /* Note! Since this is only applicable to VT-x, the implementation is placed
6624 in the VT-x part of the sources instead of the generic stuff. */
6625 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6626 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6627 return VINF_SUCCESS;
6628}
6629
6630
6631/**
6632 * Check per-VM and per-VCPU force flag actions that require us to go back to
6633 * ring-3 for one reason or another.
6634 *
6635 * @returns VBox status code (information status code included).
6636 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6637 * ring-3.
6638 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6639 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6640 * interrupts)
6641 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6642 * all EMTs to be in ring-3.
6643 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6644 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6645 * to the EM loop.
6646 *
6647 * @param pVM Pointer to the VM.
6648 * @param pVCpu Pointer to the VMCPU.
6649 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6650 * out-of-sync. Make sure to update the required fields
6651 * before using them.
6652 */
6653static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6654{
6655 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6656
6657 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6658 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6659 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6660 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6661 {
6662 /* We need the control registers now, make sure the guest-CPU context is updated. */
6663 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6664 AssertRCReturn(rc3, rc3);
6665
6666 /* Pending HM CR3 sync. */
6667 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6668 {
6669 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6670 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6671 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6672 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6673 }
6674
6675 /* Pending HM PAE PDPEs. */
6676 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6677 {
6678 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6679 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6680 }
6681
6682 /* Pending PGM C3 sync. */
6683 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6684 {
6685 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6686 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6687 if (rc2 != VINF_SUCCESS)
6688 {
6689 AssertRC(rc2);
6690 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6691 return rc2;
6692 }
6693 }
6694
6695 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6696 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6697 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6698 {
6699 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6700 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6701 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6702 return rc2;
6703 }
6704
6705 /* Pending VM request packets, such as hardware interrupts. */
6706 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6707 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6708 {
6709 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6710 return VINF_EM_PENDING_REQUEST;
6711 }
6712
6713 /* Pending PGM pool flushes. */
6714 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6715 {
6716 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6717 return VINF_PGM_POOL_FLUSH_PENDING;
6718 }
6719
6720 /* Pending DMA requests. */
6721 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6722 {
6723 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6724 return VINF_EM_RAW_TO_R3;
6725 }
6726 }
6727
6728 return VINF_SUCCESS;
6729}
6730
6731
6732/**
6733 * Converts any TRPM trap into a pending HM event. This is typically used when
6734 * entering from ring-3 (not longjmp returns).
6735 *
6736 * @param pVCpu Pointer to the VMCPU.
6737 */
6738static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6739{
6740 Assert(TRPMHasTrap(pVCpu));
6741 Assert(!pVCpu->hm.s.Event.fPending);
6742
6743 uint8_t uVector;
6744 TRPMEVENT enmTrpmEvent;
6745 RTGCUINT uErrCode;
6746 RTGCUINTPTR GCPtrFaultAddress;
6747 uint8_t cbInstr;
6748
6749 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6750 AssertRC(rc);
6751
6752 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6753 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6754 if (enmTrpmEvent == TRPM_TRAP)
6755 {
6756 switch (uVector)
6757 {
6758 case X86_XCPT_NMI:
6759 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6760 break;
6761
6762 case X86_XCPT_BP:
6763 case X86_XCPT_OF:
6764 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6765 break;
6766
6767 case X86_XCPT_PF:
6768 case X86_XCPT_DF:
6769 case X86_XCPT_TS:
6770 case X86_XCPT_NP:
6771 case X86_XCPT_SS:
6772 case X86_XCPT_GP:
6773 case X86_XCPT_AC:
6774 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6775 /* no break! */
6776 default:
6777 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6778 break;
6779 }
6780 }
6781 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6782 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6783 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6784 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6785 else
6786 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6787
6788 rc = TRPMResetTrap(pVCpu);
6789 AssertRC(rc);
6790 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6791 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6792
6793 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6794 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6795}
6796
6797
6798/**
6799 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6800 * VT-x to execute any instruction.
6801 *
6802 * @param pvCpu Pointer to the VMCPU.
6803 */
6804static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6805{
6806 Assert(pVCpu->hm.s.Event.fPending);
6807
6808 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6809 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6810 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6811 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6812
6813 /* If a trap was already pending, we did something wrong! */
6814 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6815
6816 TRPMEVENT enmTrapType;
6817 switch (uVectorType)
6818 {
6819 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6820 enmTrapType = TRPM_HARDWARE_INT;
6821 break;
6822
6823 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6824 enmTrapType = TRPM_SOFTWARE_INT;
6825 break;
6826
6827 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6828 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6829 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6830 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6831 enmTrapType = TRPM_TRAP;
6832 break;
6833
6834 default:
6835 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6836 enmTrapType = TRPM_32BIT_HACK;
6837 break;
6838 }
6839
6840 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6841
6842 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6843 AssertRC(rc);
6844
6845 if (fErrorCodeValid)
6846 TRPMSetErrorCode(pVCpu, uErrorCode);
6847
6848 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6849 && uVector == X86_XCPT_PF)
6850 {
6851 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6852 }
6853 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6854 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6855 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6856 {
6857 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6858 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6859 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6860 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6861 }
6862 pVCpu->hm.s.Event.fPending = false;
6863}
6864
6865
6866/**
6867 * Does the necessary state syncing before returning to ring-3 for any reason
6868 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6869 *
6870 * @returns VBox status code.
6871 * @param pVM Pointer to the VM.
6872 * @param pVCpu Pointer to the VMCPU.
6873 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6874 * be out-of-sync. Make sure to update the required
6875 * fields before using them.
6876 * @param fSaveGuestState Whether to save the guest state or not.
6877 *
6878 * @remarks No-long-jmp zone!!!
6879 */
6880static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6881{
6882 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6883 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6884
6885 RTCPUID idCpu = RTMpCpuId();
6886 Log4Func(("HostCpuId=%u\n", idCpu));
6887
6888 /*
6889 * !!! IMPORTANT !!!
6890 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6891 */
6892
6893 /* Save the guest state if necessary. */
6894 if ( fSaveGuestState
6895 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6896 {
6897 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6898 AssertRCReturn(rc, rc);
6899 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6900 }
6901
6902 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6903 if (CPUMIsGuestFPUStateActive(pVCpu))
6904 {
6905 /* We shouldn't reload CR0 without saving it first. */
6906 if (!fSaveGuestState)
6907 {
6908 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6909 AssertRCReturn(rc, rc);
6910 }
6911 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6912 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6913 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6914 }
6915
6916 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6917#ifdef VBOX_STRICT
6918 if (CPUMIsHyperDebugStateActive(pVCpu))
6919 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6920#endif
6921 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6922 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6923 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6924 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6925
6926#if HC_ARCH_BITS == 64
6927 /* Restore host-state bits that VT-x only restores partially. */
6928 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6929 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6930 {
6931 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6932 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6933 }
6934 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6935#endif
6936
6937#if HC_ARCH_BITS == 64
6938 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6939 if ( pVM->hm.s.fAllow64BitGuests
6940 && pVCpu->hm.s.vmx.fLazyMsrs)
6941 {
6942 /* We shouldn't reload the guest MSRs without saving it first. */
6943 if (!fSaveGuestState)
6944 {
6945 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6946 AssertRCReturn(rc, rc);
6947 }
6948 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6949 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6950 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6951 }
6952#endif
6953
6954 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6955 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6956
6957 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6958 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6959 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6960 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6961 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6962 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6963 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6964 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6965
6966 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6967
6968 /** @todo This partially defeats the purpose of having preemption hooks.
6969 * The problem is, deregistering the hooks should be moved to a place that
6970 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6971 * context.
6972 */
6973 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6974 {
6975 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6976 AssertRCReturn(rc, rc);
6977
6978 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6979 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6980 }
6981 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6982 NOREF(idCpu);
6983
6984 return VINF_SUCCESS;
6985}
6986
6987
6988/**
6989 * Leaves the VT-x session.
6990 *
6991 * @returns VBox status code.
6992 * @param pVM Pointer to the VM.
6993 * @param pVCpu Pointer to the VMCPU.
6994 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6995 * out-of-sync. Make sure to update the required fields
6996 * before using them.
6997 *
6998 * @remarks No-long-jmp zone!!!
6999 */
7000DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7001{
7002 HM_DISABLE_PREEMPT();
7003 HMVMX_ASSERT_CPU_SAFE();
7004 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7005 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7006
7007 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7008 and done this from the VMXR0ThreadCtxCallback(). */
7009 if (!pVCpu->hm.s.fLeaveDone)
7010 {
7011 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7012 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7013 pVCpu->hm.s.fLeaveDone = true;
7014 }
7015 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7016
7017 /*
7018 * !!! IMPORTANT !!!
7019 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7020 */
7021
7022 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7023 /** @todo Deregistering here means we need to VMCLEAR always
7024 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7025 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7026 VMMR0ThreadCtxHookDisable(pVCpu);
7027
7028 /* Leave HM context. This takes care of local init (term). */
7029 int rc = HMR0LeaveCpu(pVCpu);
7030
7031 HM_RESTORE_PREEMPT();
7032 return rc;
7033}
7034
7035
7036/**
7037 * Does the necessary state syncing before doing a longjmp to ring-3.
7038 *
7039 * @returns VBox status code.
7040 * @param pVM Pointer to the VM.
7041 * @param pVCpu Pointer to the VMCPU.
7042 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7043 * out-of-sync. Make sure to update the required fields
7044 * before using them.
7045 *
7046 * @remarks No-long-jmp zone!!!
7047 */
7048DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7049{
7050 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7051}
7052
7053
7054/**
7055 * Take necessary actions before going back to ring-3.
7056 *
7057 * An action requires us to go back to ring-3. This function does the necessary
7058 * steps before we can safely return to ring-3. This is not the same as longjmps
7059 * to ring-3, this is voluntary and prepares the guest so it may continue
7060 * executing outside HM (recompiler/IEM).
7061 *
7062 * @returns VBox status code.
7063 * @param pVM Pointer to the VM.
7064 * @param pVCpu Pointer to the VMCPU.
7065 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7066 * out-of-sync. Make sure to update the required fields
7067 * before using them.
7068 * @param rcExit The reason for exiting to ring-3. Can be
7069 * VINF_VMM_UNKNOWN_RING3_CALL.
7070 */
7071static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7072{
7073 Assert(pVM);
7074 Assert(pVCpu);
7075 Assert(pMixedCtx);
7076 HMVMX_ASSERT_PREEMPT_SAFE();
7077
7078 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7079 {
7080 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7081 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7082 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7083 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7084 }
7085
7086 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7087 VMMRZCallRing3Disable(pVCpu);
7088 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7089
7090 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7091 if (pVCpu->hm.s.Event.fPending)
7092 {
7093 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7094 Assert(!pVCpu->hm.s.Event.fPending);
7095 }
7096
7097 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7098 and if we're injecting an event we should have a TRPM trap pending. */
7099 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", rcExit));
7100 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", rcExit));
7101
7102 /* Save guest state and restore host state bits. */
7103 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7104 AssertRCReturn(rc, rc);
7105 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7106 /* Thread-context hooks are unregistered at this point!!! */
7107
7108 /* Sync recompiler state. */
7109 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7110 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7111 | CPUM_CHANGED_LDTR
7112 | CPUM_CHANGED_GDTR
7113 | CPUM_CHANGED_IDTR
7114 | CPUM_CHANGED_TR
7115 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7116 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7117 if ( pVM->hm.s.fNestedPaging
7118 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7119 {
7120 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7121 }
7122
7123 Assert(!pVCpu->hm.s.fClearTrapFlag);
7124
7125 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7126 if (rcExit != VINF_EM_RAW_INTERRUPT)
7127 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7128
7129 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7130
7131 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7132 VMMRZCallRing3RemoveNotification(pVCpu);
7133 VMMRZCallRing3Enable(pVCpu);
7134
7135 return rc;
7136}
7137
7138
7139/**
7140 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7141 * longjump to ring-3 and possibly get preempted.
7142 *
7143 * @returns VBox status code.
7144 * @param pVCpu Pointer to the VMCPU.
7145 * @param enmOperation The operation causing the ring-3 longjump.
7146 * @param pvUser Opaque pointer to the guest-CPU context. The data
7147 * may be out-of-sync. Make sure to update the required
7148 * fields before using them.
7149 */
7150static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7151{
7152 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7153 {
7154 /*
7155 * !!! IMPORTANT !!!
7156 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7157 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7158 */
7159 VMMRZCallRing3RemoveNotification(pVCpu);
7160 VMMRZCallRing3Disable(pVCpu);
7161 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7162 RTThreadPreemptDisable(&PreemptState);
7163
7164 PVM pVM = pVCpu->CTX_SUFF(pVM);
7165 if (CPUMIsGuestFPUStateActive(pVCpu))
7166 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7167
7168 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7169
7170#if HC_ARCH_BITS == 64
7171 /* Restore host-state bits that VT-x only restores partially. */
7172 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7173 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7174 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7175 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7176
7177 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7178 if ( pVM->hm.s.fAllow64BitGuests
7179 && pVCpu->hm.s.vmx.fLazyMsrs)
7180 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7181#endif
7182 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7183 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7184 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7185 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7186 {
7187 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7188 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7189 }
7190
7191 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7192 VMMR0ThreadCtxHookDisable(pVCpu);
7193 HMR0LeaveCpu(pVCpu);
7194 RTThreadPreemptRestore(&PreemptState);
7195 return VINF_SUCCESS;
7196 }
7197
7198 Assert(pVCpu);
7199 Assert(pvUser);
7200 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7201 HMVMX_ASSERT_PREEMPT_SAFE();
7202
7203 VMMRZCallRing3Disable(pVCpu);
7204 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7205
7206 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7207 enmOperation));
7208
7209 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7210 AssertRCReturn(rc, rc);
7211
7212 VMMRZCallRing3Enable(pVCpu);
7213 return VINF_SUCCESS;
7214}
7215
7216
7217/**
7218 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7219 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7220 *
7221 * @param pVCpu Pointer to the VMCPU.
7222 */
7223DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7224{
7225 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7226 {
7227 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7228 {
7229 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7230 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7231 AssertRC(rc);
7232 Log4(("Setup interrupt-window exiting\n"));
7233 }
7234 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7235}
7236
7237
7238/**
7239 * Clears the interrupt-window exiting control in the VMCS.
7240 *
7241 * @param pVCpu Pointer to the VMCPU.
7242 */
7243DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7244{
7245 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7246 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7247 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7248 AssertRC(rc);
7249 Log4(("Cleared interrupt-window exiting\n"));
7250}
7251
7252
7253/**
7254 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7255 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7256 *
7257 * @param pVCpu Pointer to the VMCPU.
7258 */
7259DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7260{
7261 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7262 {
7263 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7264 {
7265 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7266 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7267 AssertRC(rc);
7268 Log4(("Setup NMI-window exiting\n"));
7269 }
7270 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7271}
7272
7273
7274/**
7275 * Clears the NMI-window exiting control in the VMCS.
7276 *
7277 * @param pVCpu Pointer to the VMCPU.
7278 */
7279DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7280{
7281 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7282 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7283 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7284 AssertRC(rc);
7285 Log4(("Cleared NMI-window exiting\n"));
7286}
7287
7288
7289/**
7290 * Evaluates the event to be delivered to the guest and sets it as the pending
7291 * event.
7292 *
7293 * @param pVCpu Pointer to the VMCPU.
7294 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7295 * out-of-sync. Make sure to update the required fields
7296 * before using them.
7297 */
7298static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7299{
7300 Assert(!pVCpu->hm.s.Event.fPending);
7301
7302 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7303 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7304 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7305 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7306 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7307
7308 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7309 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7310 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7311 Assert(!TRPMHasTrap(pVCpu));
7312
7313 /*
7314 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7315 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7316 */
7317 /** @todo SMI. SMIs take priority over NMIs. */
7318 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7319 {
7320 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7321 if ( !fBlockNmi
7322 && !fBlockSti
7323 && !fBlockMovSS)
7324 {
7325 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7326 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7327 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7328
7329 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7330 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7331 }
7332 else
7333 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7334 }
7335 /*
7336 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7337 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7338 */
7339 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7340 && !pVCpu->hm.s.fSingleInstruction)
7341 {
7342 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7343 AssertRC(rc);
7344 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7345 if ( !fBlockInt
7346 && !fBlockSti
7347 && !fBlockMovSS)
7348 {
7349 uint8_t u8Interrupt;
7350 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7351 if (RT_SUCCESS(rc))
7352 {
7353 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7354 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7355 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7356
7357 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7358 }
7359 else
7360 {
7361 /** @todo Does this actually happen? If not turn it into an assertion. */
7362 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7363 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7364 }
7365 }
7366 else
7367 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7368 }
7369}
7370
7371
7372/**
7373 * Sets a pending-debug exception to be delivered to the guest if the guest is
7374 * single-stepping.
7375 *
7376 * @param pVCpu Pointer to the VMCPU.
7377 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7378 * out-of-sync. Make sure to update the required fields
7379 * before using them.
7380 */
7381DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7382{
7383 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7384 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7385 {
7386 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7387 AssertRC(rc);
7388 }
7389}
7390
7391
7392/**
7393 * Injects any pending events into the guest if the guest is in a state to
7394 * receive them.
7395 *
7396 * @returns VBox status code (informational status codes included).
7397 * @param pVCpu Pointer to the VMCPU.
7398 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7399 * out-of-sync. Make sure to update the required fields
7400 * before using them.
7401 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7402 * return VINF_EM_DBG_STEPPED if the event was
7403 * dispatched directly.
7404 */
7405static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7406{
7407 HMVMX_ASSERT_PREEMPT_SAFE();
7408 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7409
7410 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7411 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7412 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7413 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7414
7415 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7416 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7417 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7418 Assert(!TRPMHasTrap(pVCpu));
7419
7420 int rc = VINF_SUCCESS;
7421 if (pVCpu->hm.s.Event.fPending)
7422 {
7423 /*
7424 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7425 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7426 * ended up enabling interrupts outside VT-x.
7427 */
7428 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7429 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7430 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7431 {
7432 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7433 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7434 }
7435
7436#ifdef VBOX_STRICT
7437 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7438 {
7439 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7440 Assert(!fBlockInt);
7441 Assert(!fBlockSti);
7442 Assert(!fBlockMovSS);
7443 }
7444 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7445 {
7446 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7447 Assert(!fBlockSti);
7448 Assert(!fBlockMovSS);
7449 Assert(!fBlockNmi);
7450 }
7451#endif
7452 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7453 (uint8_t)uIntType));
7454 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7455 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7456 AssertRCReturn(rc, rc);
7457
7458 /* Update the interruptibility-state as it could have been changed by
7459 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7460 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7461 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7462
7463#ifdef VBOX_WITH_STATISTICS
7464 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7465 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7466 else
7467 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7468#endif
7469 }
7470
7471 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7472 if ( fBlockSti
7473 || fBlockMovSS)
7474 {
7475 if ( !pVCpu->hm.s.fSingleInstruction
7476 && !DBGFIsStepping(pVCpu))
7477 {
7478 /*
7479 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7480 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7481 * See Intel spec. 27.3.4 "Saving Non-Register State".
7482 */
7483 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7484 AssertRCReturn(rc2, rc2);
7485 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7486 }
7487 else if (pMixedCtx->eflags.Bits.u1TF)
7488 {
7489 /*
7490 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7491 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7492 */
7493 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7494 uIntrState = 0;
7495 }
7496 }
7497
7498 /*
7499 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7500 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7501 */
7502 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7503 AssertRC(rc2);
7504
7505 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7506 NOREF(fBlockMovSS); NOREF(fBlockSti);
7507 return rc;
7508}
7509
7510
7511/**
7512 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7513 *
7514 * @param pVCpu Pointer to the VMCPU.
7515 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7516 * out-of-sync. Make sure to update the required fields
7517 * before using them.
7518 */
7519DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7520{
7521 NOREF(pMixedCtx);
7522 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7523 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7524}
7525
7526
7527/**
7528 * Injects a double-fault (#DF) exception into the VM.
7529 *
7530 * @returns VBox status code (informational status code included).
7531 * @param pVCpu Pointer to the VMCPU.
7532 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7533 * out-of-sync. Make sure to update the required fields
7534 * before using them.
7535 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7536 * and should return VINF_EM_DBG_STEPPED if the event
7537 * is injected directly (register modified by us, not
7538 * by hardware on VM-entry).
7539 * @param puIntrState Pointer to the current guest interruptibility-state.
7540 * This interruptibility-state will be updated if
7541 * necessary. This cannot not be NULL.
7542 */
7543DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7544{
7545 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7546 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7547 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7548 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7549 fStepping, puIntrState);
7550}
7551
7552
7553/**
7554 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7555 *
7556 * @param pVCpu Pointer to the VMCPU.
7557 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7558 * out-of-sync. Make sure to update the required fields
7559 * before using them.
7560 */
7561DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7562{
7563 NOREF(pMixedCtx);
7564 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7565 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7566 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7567}
7568
7569
7570/**
7571 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7572 *
7573 * @param pVCpu Pointer to the VMCPU.
7574 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7575 * out-of-sync. Make sure to update the required fields
7576 * before using them.
7577 * @param cbInstr The value of RIP that is to be pushed on the guest
7578 * stack.
7579 */
7580DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7581{
7582 NOREF(pMixedCtx);
7583 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7584 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7585 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7586}
7587
7588
7589/**
7590 * Injects a general-protection (#GP) fault into the VM.
7591 *
7592 * @returns VBox status code (informational status code included).
7593 * @param pVCpu Pointer to the VMCPU.
7594 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7595 * out-of-sync. Make sure to update the required fields
7596 * before using them.
7597 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7598 * mode, i.e. in real-mode it's not valid).
7599 * @param u32ErrorCode The error code associated with the #GP.
7600 * @param fStepping Whether we're running in
7601 * hmR0VmxRunGuestCodeStep() and should return
7602 * VINF_EM_DBG_STEPPED if the event is injected
7603 * directly (register modified by us, not by
7604 * hardware on VM-entry).
7605 * @param puIntrState Pointer to the current guest interruptibility-state.
7606 * This interruptibility-state will be updated if
7607 * necessary. This cannot not be NULL.
7608 */
7609DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7610 bool fStepping, uint32_t *puIntrState)
7611{
7612 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7613 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7614 if (fErrorCodeValid)
7615 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7616 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7617 fStepping, puIntrState);
7618}
7619
7620
7621/**
7622 * Sets a general-protection (#GP) exception as pending-for-injection into the
7623 * VM.
7624 *
7625 * @param pVCpu Pointer to the VMCPU.
7626 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7627 * out-of-sync. Make sure to update the required fields
7628 * before using them.
7629 * @param u32ErrorCode The error code associated with the #GP.
7630 */
7631DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7632{
7633 NOREF(pMixedCtx);
7634 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7635 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7636 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7637 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7638}
7639
7640
7641/**
7642 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7643 *
7644 * @param pVCpu Pointer to the VMCPU.
7645 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7646 * out-of-sync. Make sure to update the required fields
7647 * before using them.
7648 * @param uVector The software interrupt vector number.
7649 * @param cbInstr The value of RIP that is to be pushed on the guest
7650 * stack.
7651 */
7652DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7653{
7654 NOREF(pMixedCtx);
7655 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7656 if ( uVector == X86_XCPT_BP
7657 || uVector == X86_XCPT_OF)
7658 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7659 else
7660 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7661 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7662}
7663
7664
7665/**
7666 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7667 * stack.
7668 *
7669 * @returns VBox status code (information status code included).
7670 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7671 * @param pVM Pointer to the VM.
7672 * @param pMixedCtx Pointer to the guest-CPU context.
7673 * @param uValue The value to push to the guest stack.
7674 */
7675DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7676{
7677 /*
7678 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7679 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7680 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7681 */
7682 if (pMixedCtx->sp == 1)
7683 return VINF_EM_RESET;
7684 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7685 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7686 AssertRCReturn(rc, rc);
7687 return rc;
7688}
7689
7690
7691/**
7692 * Injects an event into the guest upon VM-entry by updating the relevant fields
7693 * in the VM-entry area in the VMCS.
7694 *
7695 * @returns VBox status code (informational error codes included).
7696 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7697 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7698 *
7699 * @param pVCpu Pointer to the VMCPU.
7700 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7701 * be out-of-sync. Make sure to update the required
7702 * fields before using them.
7703 * @param u64IntInfo The VM-entry interruption-information field.
7704 * @param cbInstr The VM-entry instruction length in bytes (for
7705 * software interrupts, exceptions and privileged
7706 * software exceptions).
7707 * @param u32ErrCode The VM-entry exception error code.
7708 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7709 * @param puIntrState Pointer to the current guest interruptibility-state.
7710 * This interruptibility-state will be updated if
7711 * necessary. This cannot not be NULL.
7712 * @param fStepping Whether we're running in
7713 * hmR0VmxRunGuestCodeStep() and should return
7714 * VINF_EM_DBG_STEPPED if the event is injected
7715 * directly (register modified by us, not by
7716 * hardware on VM-entry).
7717 *
7718 * @remarks Requires CR0!
7719 * @remarks No-long-jump zone!!!
7720 */
7721static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7722 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7723{
7724 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7725 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7726 Assert(puIntrState);
7727 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7728
7729 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7730 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7731
7732#ifdef VBOX_STRICT
7733 /* Validate the error-code-valid bit for hardware exceptions. */
7734 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7735 {
7736 switch (uVector)
7737 {
7738 case X86_XCPT_PF:
7739 case X86_XCPT_DF:
7740 case X86_XCPT_TS:
7741 case X86_XCPT_NP:
7742 case X86_XCPT_SS:
7743 case X86_XCPT_GP:
7744 case X86_XCPT_AC:
7745 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7746 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7747 /* fallthru */
7748 default:
7749 break;
7750 }
7751 }
7752#endif
7753
7754 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7755 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7756 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7757
7758 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7759
7760 /* We require CR0 to check if the guest is in real-mode. */
7761 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7762 AssertRCReturn(rc, rc);
7763
7764 /*
7765 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7766 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7767 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7768 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7769 */
7770 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7771 {
7772 PVM pVM = pVCpu->CTX_SUFF(pVM);
7773 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7774 {
7775 Assert(PDMVmmDevHeapIsEnabled(pVM));
7776 Assert(pVM->hm.s.vmx.pRealModeTSS);
7777
7778 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7779 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7780 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7781 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7782 AssertRCReturn(rc, rc);
7783 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7784
7785 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7786 size_t const cbIdtEntry = sizeof(X86IDTR16);
7787 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7788 {
7789 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7790 if (uVector == X86_XCPT_DF)
7791 return VINF_EM_RESET;
7792
7793 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7794 if (uVector == X86_XCPT_GP)
7795 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7796
7797 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7798 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7799 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7800 fStepping, puIntrState);
7801 }
7802
7803 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7804 uint16_t uGuestIp = pMixedCtx->ip;
7805 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7806 {
7807 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7808 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7809 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7810 }
7811 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7812 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7813
7814 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7815 X86IDTR16 IdtEntry;
7816 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7817 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7818 AssertRCReturn(rc, rc);
7819
7820 /* Construct the stack frame for the interrupt/exception handler. */
7821 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7822 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7823 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7824 AssertRCReturn(rc, rc);
7825
7826 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7827 if (rc == VINF_SUCCESS)
7828 {
7829 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7830 pMixedCtx->rip = IdtEntry.offSel;
7831 pMixedCtx->cs.Sel = IdtEntry.uSel;
7832 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7833 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7834 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7835 && uVector == X86_XCPT_PF)
7836 pMixedCtx->cr2 = GCPtrFaultAddress;
7837
7838 /* If any other guest-state bits are changed here, make sure to update
7839 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7840 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7841 | HM_CHANGED_GUEST_RIP
7842 | HM_CHANGED_GUEST_RFLAGS
7843 | HM_CHANGED_GUEST_RSP);
7844
7845 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7846 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7847 {
7848 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7849 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7850 Log4(("Clearing inhibition due to STI.\n"));
7851 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7852 }
7853 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7854 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7855
7856 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7857 it, if we are returning to ring-3 before executing guest code. */
7858 pVCpu->hm.s.Event.fPending = false;
7859
7860 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7861 if (fStepping)
7862 rc = VINF_EM_DBG_STEPPED;
7863 }
7864 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7865 return rc;
7866 }
7867
7868 /*
7869 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7870 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7871 */
7872 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7873 }
7874
7875 /* Validate. */
7876 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7877 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7878 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7879
7880 /* Inject. */
7881 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7882 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7883 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7884 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7885
7886 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7887 && uVector == X86_XCPT_PF)
7888 pMixedCtx->cr2 = GCPtrFaultAddress;
7889
7890 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7891 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7892
7893 AssertRCReturn(rc, rc);
7894 return rc;
7895}
7896
7897
7898/**
7899 * Clears the interrupt-window exiting control in the VMCS and if necessary
7900 * clears the current event in the VMCS as well.
7901 *
7902 * @returns VBox status code.
7903 * @param pVCpu Pointer to the VMCPU.
7904 *
7905 * @remarks Use this function only to clear events that have not yet been
7906 * delivered to the guest but are injected in the VMCS!
7907 * @remarks No-long-jump zone!!!
7908 */
7909static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7910{
7911 int rc;
7912 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7913
7914 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7915 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7916
7917 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7918 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7919
7920 if (!pVCpu->hm.s.Event.fPending)
7921 return;
7922
7923#ifdef VBOX_STRICT
7924 uint32_t u32EntryInfo;
7925 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7926 AssertRC(rc);
7927 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7928#endif
7929
7930 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7931 AssertRC(rc);
7932
7933 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7934 AssertRC(rc);
7935
7936 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
7937 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
7938}
7939
7940
7941/**
7942 * Enters the VT-x session.
7943 *
7944 * @returns VBox status code.
7945 * @param pVM Pointer to the VM.
7946 * @param pVCpu Pointer to the VMCPU.
7947 * @param pCpu Pointer to the CPU info struct.
7948 */
7949VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7950{
7951 AssertPtr(pVM);
7952 AssertPtr(pVCpu);
7953 Assert(pVM->hm.s.vmx.fSupported);
7954 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7955 NOREF(pCpu); NOREF(pVM);
7956
7957 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7958 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7959
7960#ifdef VBOX_STRICT
7961 /* Make sure we're in VMX root mode. */
7962 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7963 if (!(u32HostCR4 & X86_CR4_VMXE))
7964 {
7965 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7966 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7967 }
7968#endif
7969
7970 /*
7971 * Load the VCPU's VMCS as the current (and active) one.
7972 */
7973 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7974 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7975 if (RT_FAILURE(rc))
7976 return rc;
7977
7978 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7979 pVCpu->hm.s.fLeaveDone = false;
7980 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7981
7982 return VINF_SUCCESS;
7983}
7984
7985
7986/**
7987 * The thread-context callback (only on platforms which support it).
7988 *
7989 * @param enmEvent The thread-context event.
7990 * @param pVCpu Pointer to the VMCPU.
7991 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7992 * @thread EMT(pVCpu)
7993 */
7994VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7995{
7996 NOREF(fGlobalInit);
7997
7998 switch (enmEvent)
7999 {
8000 case RTTHREADCTXEVENT_OUT:
8001 {
8002 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8003 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8004 VMCPU_ASSERT_EMT(pVCpu);
8005
8006 PVM pVM = pVCpu->CTX_SUFF(pVM);
8007 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8008
8009 /* No longjmps (logger flushes, locks) in this fragile context. */
8010 VMMRZCallRing3Disable(pVCpu);
8011 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8012
8013 /*
8014 * Restore host-state (FPU, debug etc.)
8015 */
8016 if (!pVCpu->hm.s.fLeaveDone)
8017 {
8018 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8019 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8020 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8021 pVCpu->hm.s.fLeaveDone = true;
8022 }
8023
8024 /* Leave HM context, takes care of local init (term). */
8025 int rc = HMR0LeaveCpu(pVCpu);
8026 AssertRC(rc); NOREF(rc);
8027
8028 /* Restore longjmp state. */
8029 VMMRZCallRing3Enable(pVCpu);
8030 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8031 break;
8032 }
8033
8034 case RTTHREADCTXEVENT_IN:
8035 {
8036 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8037 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8038 VMCPU_ASSERT_EMT(pVCpu);
8039
8040 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8041 VMMRZCallRing3Disable(pVCpu);
8042 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8043
8044 /* Initialize the bare minimum state required for HM. This takes care of
8045 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8046 int rc = HMR0EnterCpu(pVCpu);
8047 AssertRC(rc);
8048 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8049
8050 /* Load the active VMCS as the current one. */
8051 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8052 {
8053 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8054 AssertRC(rc); NOREF(rc);
8055 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8056 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8057 }
8058 pVCpu->hm.s.fLeaveDone = false;
8059
8060 /* Restore longjmp state. */
8061 VMMRZCallRing3Enable(pVCpu);
8062 break;
8063 }
8064
8065 default:
8066 break;
8067 }
8068}
8069
8070
8071/**
8072 * Saves the host state in the VMCS host-state.
8073 * Sets up the VM-exit MSR-load area.
8074 *
8075 * The CPU state will be loaded from these fields on every successful VM-exit.
8076 *
8077 * @returns VBox status code.
8078 * @param pVM Pointer to the VM.
8079 * @param pVCpu Pointer to the VMCPU.
8080 *
8081 * @remarks No-long-jump zone!!!
8082 */
8083static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8084{
8085 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8086
8087 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8088 return VINF_SUCCESS;
8089
8090 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8091 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8092
8093 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8094 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8095
8096 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8097 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8098
8099 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8100 return rc;
8101}
8102
8103
8104/**
8105 * Saves the host state in the VMCS host-state.
8106 *
8107 * @returns VBox status code.
8108 * @param pVM Pointer to the VM.
8109 * @param pVCpu Pointer to the VMCPU.
8110 *
8111 * @remarks No-long-jump zone!!!
8112 */
8113VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8114{
8115 AssertPtr(pVM);
8116 AssertPtr(pVCpu);
8117
8118 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8119
8120 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8121 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8122 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8123 return hmR0VmxSaveHostState(pVM, pVCpu);
8124}
8125
8126
8127/**
8128 * Loads the guest state into the VMCS guest-state area.
8129 *
8130 * The will typically be done before VM-entry when the guest-CPU state and the
8131 * VMCS state may potentially be out of sync.
8132 *
8133 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8134 * VM-entry controls.
8135 * Sets up the appropriate VMX non-root function to execute guest code based on
8136 * the guest CPU mode.
8137 *
8138 * @returns VBox status code.
8139 * @param pVM Pointer to the VM.
8140 * @param pVCpu Pointer to the VMCPU.
8141 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8142 * out-of-sync. Make sure to update the required fields
8143 * before using them.
8144 *
8145 * @remarks No-long-jump zone!!!
8146 */
8147static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8148{
8149 AssertPtr(pVM);
8150 AssertPtr(pVCpu);
8151 AssertPtr(pMixedCtx);
8152 HMVMX_ASSERT_PREEMPT_SAFE();
8153
8154 VMMRZCallRing3Disable(pVCpu);
8155 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8156
8157 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8158
8159 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8160
8161 /* Determine real-on-v86 mode. */
8162 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8163 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8164 && CPUMIsGuestInRealModeEx(pMixedCtx))
8165 {
8166 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8167 }
8168
8169 /*
8170 * Load the guest-state into the VMCS.
8171 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8172 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8173 */
8174 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8175 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8176
8177 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8178 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8179 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8180
8181 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8182 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8183 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8184
8185 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8186 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8187
8188 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8189 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8190
8191 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8192 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8193 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8194
8195 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8196 determine we don't have to swap EFER after all. */
8197 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8198 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8199
8200 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8201 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8202
8203 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8204 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8205
8206 /*
8207 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8208 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8209 */
8210 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8211 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8212
8213 /* Clear any unused and reserved bits. */
8214 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8215
8216 VMMRZCallRing3Enable(pVCpu);
8217
8218 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8219 return rc;
8220}
8221
8222
8223/**
8224 * Loads the state shared between the host and guest into the VMCS.
8225 *
8226 * @param pVM Pointer to the VM.
8227 * @param pVCpu Pointer to the VMCPU.
8228 * @param pCtx Pointer to the guest-CPU context.
8229 *
8230 * @remarks No-long-jump zone!!!
8231 */
8232static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8233{
8234 NOREF(pVM);
8235
8236 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8237 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8238
8239 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8240 {
8241 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8242 AssertRC(rc);
8243 }
8244
8245 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8246 {
8247 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8248 AssertRC(rc);
8249
8250 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8251 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8252 {
8253 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8254 AssertRC(rc);
8255 }
8256 }
8257
8258 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8259 {
8260#if HC_ARCH_BITS == 64
8261 if (pVM->hm.s.fAllow64BitGuests)
8262 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8263#endif
8264 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8265 }
8266
8267 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8268 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8269 {
8270 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8271 AssertRC(rc);
8272 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8273 }
8274
8275 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8276 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8277}
8278
8279
8280/**
8281 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8282 *
8283 * @param pVM Pointer to the VM.
8284 * @param pVCpu Pointer to the VMCPU.
8285 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8286 * out-of-sync. Make sure to update the required fields
8287 * before using them.
8288 */
8289DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8290{
8291 HMVMX_ASSERT_PREEMPT_SAFE();
8292
8293 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8294#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8295 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8296#endif
8297
8298 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8299 {
8300 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8301 AssertRC(rc);
8302 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8303 }
8304 else if (HMCPU_CF_VALUE(pVCpu))
8305 {
8306 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8307 AssertRC(rc);
8308 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8309 }
8310
8311 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8312 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8313 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8314 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8315}
8316
8317
8318/**
8319 * Does the preparations before executing guest code in VT-x.
8320 *
8321 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8322 * recompiler/IEM. We must be cautious what we do here regarding committing
8323 * guest-state information into the VMCS assuming we assuredly execute the
8324 * guest in VT-x mode.
8325 *
8326 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8327 * the common-state (TRPM/forceflags), we must undo those changes so that the
8328 * recompiler/IEM can (and should) use them when it resumes guest execution.
8329 * Otherwise such operations must be done when we can no longer exit to ring-3.
8330 *
8331 * @returns Strict VBox status code.
8332 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8333 * have been disabled.
8334 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8335 * double-fault into the guest.
8336 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8337 * dispatched directly.
8338 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8339 *
8340 * @param pVM Pointer to the VM.
8341 * @param pVCpu Pointer to the VMCPU.
8342 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8343 * out-of-sync. Make sure to update the required fields
8344 * before using them.
8345 * @param pVmxTransient Pointer to the VMX transient structure.
8346 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8347 * us ignore some of the reasons for returning to
8348 * ring-3, and return VINF_EM_DBG_STEPPED if event
8349 * dispatching took place.
8350 */
8351static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8352{
8353 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8354
8355#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8356 PGMRZDynMapFlushAutoSet(pVCpu);
8357#endif
8358
8359 /* Check force flag actions that might require us to go back to ring-3. */
8360 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8361 if (rc != VINF_SUCCESS)
8362 return rc;
8363
8364#ifndef IEM_VERIFICATION_MODE_FULL
8365 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8366 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8367 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8368 {
8369 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8370 RTGCPHYS GCPhysApicBase;
8371 GCPhysApicBase = pMixedCtx->msrApicBase;
8372 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8373
8374 /* Unalias any existing mapping. */
8375 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8376 AssertRCReturn(rc, rc);
8377
8378 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8379 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8380 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8381 AssertRCReturn(rc, rc);
8382
8383 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8384 }
8385#endif /* !IEM_VERIFICATION_MODE_FULL */
8386
8387 if (TRPMHasTrap(pVCpu))
8388 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8389 else if (!pVCpu->hm.s.Event.fPending)
8390 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8391
8392 /*
8393 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8394 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8395 */
8396 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8397 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8398 {
8399 Assert(rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
8400 return rc;
8401 }
8402
8403 /*
8404 * Load the guest state bits, we can handle longjmps/getting preempted here.
8405 *
8406 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8407 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8408 * Hence, this needs to be done -after- injection of events.
8409 */
8410 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8411
8412 /*
8413 * No longjmps to ring-3 from this point on!!!
8414 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8415 * This also disables flushing of the R0-logger instance (if any).
8416 */
8417 VMMRZCallRing3Disable(pVCpu);
8418
8419 /*
8420 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8421 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8422 *
8423 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8424 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8425 *
8426 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8427 * executing guest code.
8428 */
8429 pVmxTransient->fEFlags = ASMIntDisableFlags();
8430 if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8431 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8432 && ( !fStepping /* Optimized for the non-stepping case, of course. */
8433 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8434 {
8435 hmR0VmxClearEventVmcs(pVCpu);
8436 ASMSetFlags(pVmxTransient->fEFlags);
8437 VMMRZCallRing3Enable(pVCpu);
8438 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8439 return VINF_EM_RAW_TO_R3;
8440 }
8441
8442 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8443 {
8444 hmR0VmxClearEventVmcs(pVCpu);
8445 ASMSetFlags(pVmxTransient->fEFlags);
8446 VMMRZCallRing3Enable(pVCpu);
8447 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8448 return VINF_EM_RAW_INTERRUPT;
8449 }
8450
8451 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8452 pVCpu->hm.s.Event.fPending = false;
8453
8454 return VINF_SUCCESS;
8455}
8456
8457
8458/**
8459 * Prepares to run guest code in VT-x and we've committed to doing so. This
8460 * means there is no backing out to ring-3 or anywhere else at this
8461 * point.
8462 *
8463 * @param pVM Pointer to the VM.
8464 * @param pVCpu Pointer to the VMCPU.
8465 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8466 * out-of-sync. Make sure to update the required fields
8467 * before using them.
8468 * @param pVmxTransient Pointer to the VMX transient structure.
8469 *
8470 * @remarks Called with preemption disabled.
8471 * @remarks No-long-jump zone!!!
8472 */
8473static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8474{
8475 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8476 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8477 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8478
8479 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8480 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8481
8482#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8483 if (!CPUMIsGuestFPUStateActive(pVCpu))
8484 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8485 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8486#endif
8487
8488 if ( pVCpu->hm.s.fPreloadGuestFpu
8489 && !CPUMIsGuestFPUStateActive(pVCpu))
8490 {
8491 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8492 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8493 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8494 }
8495
8496 /*
8497 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8498 */
8499 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8500 && pVCpu->hm.s.vmx.cMsrs > 0)
8501 {
8502 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8503 }
8504
8505 /*
8506 * Load the host state bits as we may've been preempted (only happens when
8507 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8508 */
8509 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8510 * any effect to the host state needing to be saved? */
8511 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8512 {
8513 /* This ASSUMES that pfnStartVM has been set up already. */
8514 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8515 AssertRC(rc);
8516 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8517 }
8518 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8519
8520 /*
8521 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8522 */
8523 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8524 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8525 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8526
8527 /* Store status of the shared guest-host state at the time of VM-entry. */
8528#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8529 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8530 {
8531 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8532 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8533 }
8534 else
8535#endif
8536 {
8537 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8538 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8539 }
8540 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8541
8542 /*
8543 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8544 */
8545 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8546 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8547
8548 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8549 RTCPUID idCurrentCpu = pCpu->idCpu;
8550 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8551 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8552 {
8553 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8554 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8555 }
8556
8557 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8558 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8559 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8560 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8561
8562 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8563
8564 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8565 to start executing. */
8566
8567 /*
8568 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8569 */
8570 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8571 {
8572 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8573 {
8574 bool fMsrUpdated;
8575 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8576 AssertRC(rc2);
8577 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8578
8579 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8580 &fMsrUpdated);
8581 AssertRC(rc2);
8582 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8583
8584 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8585 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8586 }
8587 else
8588 {
8589 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8590 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8591 }
8592 }
8593
8594#ifdef VBOX_STRICT
8595 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8596 hmR0VmxCheckHostEferMsr(pVCpu);
8597 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8598#endif
8599#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8600 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8601 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8602 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8603#endif
8604}
8605
8606
8607/**
8608 * Performs some essential restoration of state after running guest code in
8609 * VT-x.
8610 *
8611 * @param pVM Pointer to the VM.
8612 * @param pVCpu Pointer to the VMCPU.
8613 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8614 * out-of-sync. Make sure to update the required fields
8615 * before using them.
8616 * @param pVmxTransient Pointer to the VMX transient structure.
8617 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8618 *
8619 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8620 *
8621 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8622 * unconditionally when it is safe to do so.
8623 */
8624static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8625{
8626 NOREF(pVM);
8627
8628 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8629
8630 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8631 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8632 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8633 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8634 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8635 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8636
8637 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8638 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8639
8640 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8641 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8642 Assert(!ASMIntAreEnabled());
8643 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8644
8645#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8646 if (CPUMIsGuestFPUStateActive(pVCpu))
8647 {
8648 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8649 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8650 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8651 }
8652#endif
8653
8654#if HC_ARCH_BITS == 64
8655 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8656#endif
8657 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8658#ifdef VBOX_STRICT
8659 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8660#endif
8661 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8662 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8663
8664 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8665 uint32_t uExitReason;
8666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8667 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8668 AssertRC(rc);
8669 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8670 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8671
8672 /* Update the VM-exit history array. */
8673 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8674
8675 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8676 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8677 {
8678 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8679 pVmxTransient->fVMEntryFailed));
8680 return;
8681 }
8682
8683 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8684 {
8685 /** @todo We can optimize this by only syncing with our force-flags when
8686 * really needed and keeping the VMCS state as it is for most
8687 * VM-exits. */
8688 /* Update the guest interruptibility-state from the VMCS. */
8689 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8690
8691#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8692 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8693 AssertRC(rc);
8694#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8695 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8696 AssertRC(rc);
8697#endif
8698
8699 /*
8700 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8701 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8702 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8703 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8704 */
8705 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8706 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8707 {
8708 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8709 AssertRC(rc);
8710 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8711 }
8712 }
8713}
8714
8715
8716/**
8717 * Runs the guest code using VT-x the normal way.
8718 *
8719 * @returns VBox status code.
8720 * @param pVM Pointer to the VM.
8721 * @param pVCpu Pointer to the VMCPU.
8722 * @param pCtx Pointer to the guest-CPU context.
8723 *
8724 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8725 */
8726static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8727{
8728 VMXTRANSIENT VmxTransient;
8729 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8730 int rc = VERR_INTERNAL_ERROR_5;
8731 uint32_t cLoops = 0;
8732
8733 for (;; cLoops++)
8734 {
8735 Assert(!HMR0SuspendPending());
8736 HMVMX_ASSERT_CPU_SAFE();
8737
8738 /* Preparatory work for running guest code, this may force us to return
8739 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8740 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8741 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8742 if (rc != VINF_SUCCESS)
8743 break;
8744
8745 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8746 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8747 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8748
8749 /* Restore any residual host-state and save any bits shared between host
8750 and guest into the guest-CPU state. Re-enables interrupts! */
8751 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8752
8753 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8754 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8755 {
8756 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8757 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8758 return rc;
8759 }
8760
8761 /* Profile the VM-exit. */
8762 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8763 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8764 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8765 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8766 HMVMX_START_EXIT_DISPATCH_PROF();
8767
8768 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8769 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8770 {
8771 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8772 hmR0VmxSaveGuestState(pVCpu, pCtx);
8773 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8774 }
8775
8776 /* Handle the VM-exit. */
8777#ifdef HMVMX_USE_FUNCTION_TABLE
8778 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8779#else
8780 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8781#endif
8782 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8783 if (rc != VINF_SUCCESS)
8784 break;
8785 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8786 {
8787 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8788 rc = VINF_EM_RAW_INTERRUPT;
8789 break;
8790 }
8791 }
8792
8793 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8794 return rc;
8795}
8796
8797
8798/**
8799 * Single steps guest code using VT-x.
8800 *
8801 * @returns VBox status code.
8802 * @param pVM Pointer to the VM.
8803 * @param pVCpu Pointer to the VMCPU.
8804 * @param pCtx Pointer to the guest-CPU context.
8805 *
8806 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8807 */
8808static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8809{
8810 VMXTRANSIENT VmxTransient;
8811 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8812 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8813 uint32_t cLoops = 0;
8814 uint16_t uCsStart = pCtx->cs.Sel;
8815 uint64_t uRipStart = pCtx->rip;
8816
8817 for (;; cLoops++)
8818 {
8819 Assert(!HMR0SuspendPending());
8820 HMVMX_ASSERT_CPU_SAFE();
8821
8822 /* Preparatory work for running guest code, this may force us to return
8823 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8824 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8825 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8826 if (rcStrict != VINF_SUCCESS)
8827 break;
8828
8829 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8830 rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8831 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8832
8833 /* Restore any residual host-state and save any bits shared between host
8834 and guest into the guest-CPU state. Re-enables interrupts! */
8835 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8836
8837 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8838 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
8839 {
8840 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8841 hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
8842 return VBOXSTRICTRC_TODO(rcStrict);
8843 }
8844
8845 /* Profile the VM-exit. */
8846 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8847 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8848 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8849 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8850 HMVMX_START_EXIT_DISPATCH_PROF();
8851
8852 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8853 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8854 {
8855 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8856 hmR0VmxSaveGuestState(pVCpu, pCtx);
8857 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8858 }
8859
8860 /* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8861 rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8862 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8863 if (rcStrict != VINF_SUCCESS)
8864 break;
8865 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8866 {
8867 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8868 rcStrict = VINF_EM_RAW_INTERRUPT;
8869 break;
8870 }
8871
8872 /*
8873 * Did the RIP change, if so, consider it a single step.
8874 * Otherwise, make sure one of the TFs gets set.
8875 */
8876 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8877 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8878 AssertRCReturn(rc2, rc2);
8879 if ( pCtx->rip != uRipStart
8880 || pCtx->cs.Sel != uCsStart)
8881 {
8882 rcStrict = VINF_EM_DBG_STEPPED;
8883 break;
8884 }
8885 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8886 }
8887
8888 /*
8889 * Clear the X86_EFL_TF if necessary.
8890 */
8891 if (pVCpu->hm.s.fClearTrapFlag)
8892 {
8893 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8894 AssertRCReturn(rc2, rc2);
8895 pVCpu->hm.s.fClearTrapFlag = false;
8896 pCtx->eflags.Bits.u1TF = 0;
8897 }
8898 /** @todo there seems to be issues with the resume flag when the monitor trap
8899 * flag is pending without being used. Seen early in bios init when
8900 * accessing APIC page in protected mode. */
8901
8902 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8903 return VBOXSTRICTRC_TODO(rcStrict);
8904}
8905
8906
8907/**
8908 * Runs the guest code using VT-x.
8909 *
8910 * @returns VBox status code.
8911 * @param pVM Pointer to the VM.
8912 * @param pVCpu Pointer to the VMCPU.
8913 * @param pCtx Pointer to the guest-CPU context.
8914 */
8915VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8916{
8917 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8918 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8919 HMVMX_ASSERT_PREEMPT_SAFE();
8920
8921 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8922
8923 int rc;
8924 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8925 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8926 else
8927 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8928
8929 if (rc == VERR_EM_INTERPRETER)
8930 rc = VINF_EM_RAW_EMULATE_INSTR;
8931 else if (rc == VINF_EM_RESET)
8932 rc = VINF_EM_TRIPLE_FAULT;
8933
8934 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8935 if (RT_FAILURE(rc2))
8936 {
8937 pVCpu->hm.s.u32HMError = rc;
8938 rc = rc2;
8939 }
8940 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8941 return rc;
8942}
8943
8944
8945#ifndef HMVMX_USE_FUNCTION_TABLE
8946DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8947{
8948#ifdef DEBUG_ramshankar
8949# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8950# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8951#endif
8952 int rc;
8953 switch (rcReason)
8954 {
8955 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8956 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8957 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8958 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8959 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8960 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8961 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8962 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8963 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8964 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8965 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8966 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8967 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8968 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8969 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8970 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8971 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8972 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8973 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8974 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8975 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8976 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8977 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8978 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8979 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8980 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8981 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8982 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8983 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8984 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8985 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8986 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8987 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8988 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8989
8990 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8991 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8992 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8993 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8994 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8995 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8996 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8997 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8998 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8999
9000 case VMX_EXIT_VMCLEAR:
9001 case VMX_EXIT_VMLAUNCH:
9002 case VMX_EXIT_VMPTRLD:
9003 case VMX_EXIT_VMPTRST:
9004 case VMX_EXIT_VMREAD:
9005 case VMX_EXIT_VMRESUME:
9006 case VMX_EXIT_VMWRITE:
9007 case VMX_EXIT_VMXOFF:
9008 case VMX_EXIT_VMXON:
9009 case VMX_EXIT_INVEPT:
9010 case VMX_EXIT_INVVPID:
9011 case VMX_EXIT_VMFUNC:
9012 case VMX_EXIT_XSAVES:
9013 case VMX_EXIT_XRSTORS:
9014 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9015 break;
9016 case VMX_EXIT_RESERVED_60:
9017 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
9018 case VMX_EXIT_RESERVED_62:
9019 default:
9020 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9021 break;
9022 }
9023 return rc;
9024}
9025#endif /* !HMVMX_USE_FUNCTION_TABLE */
9026
9027
9028/**
9029 * Single-stepping VM-exit filtering.
9030 *
9031 * This is preprocessing the exits and deciding whether we've gotten far enough
9032 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9033 * performed.
9034 *
9035 * @returns Strict VBox status code.
9036 * @param pVCpu The virtual CPU of the calling EMT.
9037 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9038 * out-of-sync. Make sure to update the required
9039 * fields before using them.
9040 * @param pVmxTransient Pointer to the VMX-transient structure.
9041 * @param uExitReason The VM-exit reason.
9042 */
9043DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9044 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9045{
9046 switch (uExitReason)
9047 {
9048 case VMX_EXIT_XCPT_OR_NMI:
9049 {
9050 /* Check for host NMI. */
9051 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9052 AssertRCReturn(rc2, rc2);
9053 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9054 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9055 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9056 /* fall thru */
9057 }
9058
9059 case VMX_EXIT_EPT_MISCONFIG:
9060 case VMX_EXIT_TRIPLE_FAULT:
9061 case VMX_EXIT_APIC_ACCESS:
9062 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9063 case VMX_EXIT_TASK_SWITCH:
9064
9065 /* Instruction specific VM-exits: */
9066 case VMX_EXIT_IO_INSTR:
9067 case VMX_EXIT_CPUID:
9068 case VMX_EXIT_RDTSC:
9069 case VMX_EXIT_RDTSCP:
9070 case VMX_EXIT_MOV_CRX:
9071 case VMX_EXIT_MWAIT:
9072 case VMX_EXIT_MONITOR:
9073 case VMX_EXIT_RDMSR:
9074 case VMX_EXIT_WRMSR:
9075 case VMX_EXIT_MOV_DRX:
9076 case VMX_EXIT_HLT:
9077 case VMX_EXIT_INVD:
9078 case VMX_EXIT_INVLPG:
9079 case VMX_EXIT_RSM:
9080 case VMX_EXIT_PAUSE:
9081 case VMX_EXIT_XDTR_ACCESS:
9082 case VMX_EXIT_TR_ACCESS:
9083 case VMX_EXIT_WBINVD:
9084 case VMX_EXIT_XSETBV:
9085 case VMX_EXIT_RDRAND:
9086 case VMX_EXIT_INVPCID:
9087 case VMX_EXIT_GETSEC:
9088 case VMX_EXIT_RDPMC:
9089 case VMX_EXIT_VMCALL:
9090 case VMX_EXIT_VMCLEAR:
9091 case VMX_EXIT_VMLAUNCH:
9092 case VMX_EXIT_VMPTRLD:
9093 case VMX_EXIT_VMPTRST:
9094 case VMX_EXIT_VMREAD:
9095 case VMX_EXIT_VMRESUME:
9096 case VMX_EXIT_VMWRITE:
9097 case VMX_EXIT_VMXOFF:
9098 case VMX_EXIT_VMXON:
9099 case VMX_EXIT_INVEPT:
9100 case VMX_EXIT_INVVPID:
9101 case VMX_EXIT_VMFUNC:
9102 {
9103 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9104 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9105 AssertRCReturn(rc2, rc2);
9106 if ( pMixedCtx->rip != uRipStart
9107 || pMixedCtx->cs.Sel != uCsStart)
9108 return VINF_EM_DBG_STEPPED;
9109 break;
9110 }
9111 }
9112
9113 /*
9114 * Normal processing.
9115 */
9116#ifdef HMVMX_USE_FUNCTION_TABLE
9117 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9118#else
9119 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9120#endif
9121}
9122
9123
9124#ifdef DEBUG
9125/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9126# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9127 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9128
9129# define HMVMX_ASSERT_PREEMPT_CPUID() \
9130 do \
9131 { \
9132 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9133 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9134 } while (0)
9135
9136# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9137 do { \
9138 AssertPtr(pVCpu); \
9139 AssertPtr(pMixedCtx); \
9140 AssertPtr(pVmxTransient); \
9141 Assert(pVmxTransient->fVMEntryFailed == false); \
9142 Assert(ASMIntAreEnabled()); \
9143 HMVMX_ASSERT_PREEMPT_SAFE(); \
9144 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9145 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9146 HMVMX_ASSERT_PREEMPT_SAFE(); \
9147 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9148 HMVMX_ASSERT_PREEMPT_CPUID(); \
9149 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9150 } while (0)
9151
9152# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9153 do { \
9154 Log4Func(("\n")); \
9155 } while (0)
9156#else /* Release builds */
9157# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9158 do { \
9159 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9160 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9161 } while (0)
9162# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9163#endif
9164
9165
9166/**
9167 * Advances the guest RIP after reading it from the VMCS.
9168 *
9169 * @returns VBox status code.
9170 * @param pVCpu Pointer to the VMCPU.
9171 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9172 * out-of-sync. Make sure to update the required fields
9173 * before using them.
9174 * @param pVmxTransient Pointer to the VMX transient structure.
9175 *
9176 * @remarks No-long-jump zone!!!
9177 */
9178DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9179{
9180 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9181 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9182 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9183 AssertRCReturn(rc, rc);
9184
9185 pMixedCtx->rip += pVmxTransient->cbInstr;
9186 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9187
9188 /*
9189 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9190 * pending debug exception field as it takes care of priority of events.
9191 *
9192 * See Intel spec. 32.2.1 "Debug Exceptions".
9193 */
9194 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9195
9196 return rc;
9197}
9198
9199
9200/**
9201 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9202 * and update error record fields accordingly.
9203 *
9204 * @return VMX_IGS_* return codes.
9205 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9206 * wrong with the guest state.
9207 *
9208 * @param pVM Pointer to the VM.
9209 * @param pVCpu Pointer to the VMCPU.
9210 * @param pCtx Pointer to the guest-CPU state.
9211 *
9212 * @remarks This function assumes our cache of the VMCS controls
9213 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9214 */
9215static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9216{
9217#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9218#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9219 uError = (err); \
9220 break; \
9221 } else do { } while (0)
9222
9223 int rc;
9224 uint32_t uError = VMX_IGS_ERROR;
9225 uint32_t u32Val;
9226 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9227
9228 do
9229 {
9230 /*
9231 * CR0.
9232 */
9233 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9234 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9235 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9236 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9237 if (fUnrestrictedGuest)
9238 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9239
9240 uint32_t u32GuestCR0;
9241 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9242 AssertRCBreak(rc);
9243 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9244 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9245 if ( !fUnrestrictedGuest
9246 && (u32GuestCR0 & X86_CR0_PG)
9247 && !(u32GuestCR0 & X86_CR0_PE))
9248 {
9249 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9250 }
9251
9252 /*
9253 * CR4.
9254 */
9255 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9256 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9257
9258 uint32_t u32GuestCR4;
9259 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9260 AssertRCBreak(rc);
9261 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9262 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9263
9264 /*
9265 * IA32_DEBUGCTL MSR.
9266 */
9267 uint64_t u64Val;
9268 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9269 AssertRCBreak(rc);
9270 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9271 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9272 {
9273 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9274 }
9275 uint64_t u64DebugCtlMsr = u64Val;
9276
9277#ifdef VBOX_STRICT
9278 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9279 AssertRCBreak(rc);
9280 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9281#endif
9282 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9283
9284 /*
9285 * RIP and RFLAGS.
9286 */
9287 uint32_t u32Eflags;
9288#if HC_ARCH_BITS == 64
9289 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9290 AssertRCBreak(rc);
9291 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9292 if ( !fLongModeGuest
9293 || !pCtx->cs.Attr.n.u1Long)
9294 {
9295 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9296 }
9297 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9298 * must be identical if the "IA-32e mode guest" VM-entry
9299 * control is 1 and CS.L is 1. No check applies if the
9300 * CPU supports 64 linear-address bits. */
9301
9302 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9303 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9304 AssertRCBreak(rc);
9305 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9306 VMX_IGS_RFLAGS_RESERVED);
9307 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9308 u32Eflags = u64Val;
9309#else
9310 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9311 AssertRCBreak(rc);
9312 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9313 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9314#endif
9315
9316 if ( fLongModeGuest
9317 || ( fUnrestrictedGuest
9318 && !(u32GuestCR0 & X86_CR0_PE)))
9319 {
9320 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9321 }
9322
9323 uint32_t u32EntryInfo;
9324 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9325 AssertRCBreak(rc);
9326 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9327 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9328 {
9329 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9330 }
9331
9332 /*
9333 * 64-bit checks.
9334 */
9335#if HC_ARCH_BITS == 64
9336 if (fLongModeGuest)
9337 {
9338 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9339 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9340 }
9341
9342 if ( !fLongModeGuest
9343 && (u32GuestCR4 & X86_CR4_PCIDE))
9344 {
9345 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9346 }
9347
9348 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9349 * 51:32 beyond the processor's physical-address width are 0. */
9350
9351 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9352 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9353 {
9354 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9355 }
9356
9357 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9358 AssertRCBreak(rc);
9359 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9360
9361 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9362 AssertRCBreak(rc);
9363 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9364#endif
9365
9366 /*
9367 * PERF_GLOBAL MSR.
9368 */
9369 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9370 {
9371 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9372 AssertRCBreak(rc);
9373 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9374 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9375 }
9376
9377 /*
9378 * PAT MSR.
9379 */
9380 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9381 {
9382 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9383 AssertRCBreak(rc);
9384 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9385 for (unsigned i = 0; i < 8; i++)
9386 {
9387 uint8_t u8Val = (u64Val & 0xff);
9388 if ( u8Val != 0 /* UC */
9389 && u8Val != 1 /* WC */
9390 && u8Val != 4 /* WT */
9391 && u8Val != 5 /* WP */
9392 && u8Val != 6 /* WB */
9393 && u8Val != 7 /* UC- */)
9394 {
9395 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9396 }
9397 u64Val >>= 8;
9398 }
9399 }
9400
9401 /*
9402 * EFER MSR.
9403 */
9404 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9405 {
9406 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9407 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9408 AssertRCBreak(rc);
9409 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9410 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9411 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
9412 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9413 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9414 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9415 || !(u32GuestCR0 & X86_CR0_PG)
9416 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9417 VMX_IGS_EFER_LMA_LME_MISMATCH);
9418 }
9419
9420 /*
9421 * Segment registers.
9422 */
9423 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9424 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9425 if (!(u32Eflags & X86_EFL_VM))
9426 {
9427 /* CS */
9428 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9429 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9430 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9431 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9432 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9433 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9434 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9435 /* CS cannot be loaded with NULL in protected mode. */
9436 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9437 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9438 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9439 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9440 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9441 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9442 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9443 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9444 else
9445 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9446
9447 /* SS */
9448 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9449 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9450 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9451 if ( !(pCtx->cr0 & X86_CR0_PE)
9452 || pCtx->cs.Attr.n.u4Type == 3)
9453 {
9454 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9455 }
9456 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9457 {
9458 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9459 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9460 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9461 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9462 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9463 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9464 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9465 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9466 }
9467
9468 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9469 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9470 {
9471 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9472 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9473 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9474 || pCtx->ds.Attr.n.u4Type > 11
9475 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9476 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9477 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9478 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9479 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9480 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9481 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9482 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9483 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9484 }
9485 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9486 {
9487 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9488 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9489 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9490 || pCtx->es.Attr.n.u4Type > 11
9491 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9492 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9493 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9494 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9495 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9496 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9497 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9498 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9499 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9500 }
9501 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9502 {
9503 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9504 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9505 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9506 || pCtx->fs.Attr.n.u4Type > 11
9507 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9508 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9509 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9510 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9511 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9512 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9513 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9514 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9515 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9516 }
9517 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9518 {
9519 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9520 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9521 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9522 || pCtx->gs.Attr.n.u4Type > 11
9523 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9524 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9525 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9526 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9527 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9528 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9529 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9530 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9531 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9532 }
9533 /* 64-bit capable CPUs. */
9534#if HC_ARCH_BITS == 64
9535 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9536 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9537 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9538 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9539 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9540 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9541 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9542 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9543 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9544 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9545 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9546#endif
9547 }
9548 else
9549 {
9550 /* V86 mode checks. */
9551 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9552 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9553 {
9554 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9555 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9556 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9557 }
9558 else
9559 {
9560 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9561 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9562 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9563 }
9564
9565 /* CS */
9566 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9567 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9568 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9569 /* SS */
9570 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9571 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9572 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9573 /* DS */
9574 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9575 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9576 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9577 /* ES */
9578 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9579 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9580 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9581 /* FS */
9582 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9583 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9584 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9585 /* GS */
9586 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9587 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9588 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9589 /* 64-bit capable CPUs. */
9590#if HC_ARCH_BITS == 64
9591 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9592 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9593 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9594 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9595 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9596 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9597 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9598 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9599 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9600 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9601 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9602#endif
9603 }
9604
9605 /*
9606 * TR.
9607 */
9608 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9609 /* 64-bit capable CPUs. */
9610#if HC_ARCH_BITS == 64
9611 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9612#endif
9613 if (fLongModeGuest)
9614 {
9615 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9616 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9617 }
9618 else
9619 {
9620 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9621 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9622 VMX_IGS_TR_ATTR_TYPE_INVALID);
9623 }
9624 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9625 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9626 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9627 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9628 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9629 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9630 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9631 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9632
9633 /*
9634 * GDTR and IDTR.
9635 */
9636#if HC_ARCH_BITS == 64
9637 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9638 AssertRCBreak(rc);
9639 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9640
9641 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9642 AssertRCBreak(rc);
9643 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9644#endif
9645
9646 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9647 AssertRCBreak(rc);
9648 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9649
9650 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9651 AssertRCBreak(rc);
9652 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9653
9654 /*
9655 * Guest Non-Register State.
9656 */
9657 /* Activity State. */
9658 uint32_t u32ActivityState;
9659 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9660 AssertRCBreak(rc);
9661 HMVMX_CHECK_BREAK( !u32ActivityState
9662 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9663 VMX_IGS_ACTIVITY_STATE_INVALID);
9664 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9665 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9666 uint32_t u32IntrState;
9667 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9668 AssertRCBreak(rc);
9669 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9670 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9671 {
9672 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9673 }
9674
9675 /** @todo Activity state and injecting interrupts. Left as a todo since we
9676 * currently don't use activity states but ACTIVE. */
9677
9678 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9679 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9680
9681 /* Guest interruptibility-state. */
9682 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9683 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9684 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9685 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9686 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9687 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9688 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9689 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9690 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9691 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9692 {
9693 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9694 {
9695 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9696 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9697 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9698 }
9699 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9700 {
9701 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9702 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9703 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9704 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9705 }
9706 }
9707 /** @todo Assumes the processor is not in SMM. */
9708 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9709 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9710 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9711 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9712 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9713 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9714 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9715 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9716 {
9717 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9718 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9719 }
9720
9721 /* Pending debug exceptions. */
9722#if HC_ARCH_BITS == 64
9723 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9724 AssertRCBreak(rc);
9725 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9726 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9727 u32Val = u64Val; /* For pending debug exceptions checks below. */
9728#else
9729 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9730 AssertRCBreak(rc);
9731 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9732 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9733#endif
9734
9735 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9736 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9737 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9738 {
9739 if ( (u32Eflags & X86_EFL_TF)
9740 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9741 {
9742 /* Bit 14 is PendingDebug.BS. */
9743 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9744 }
9745 if ( !(u32Eflags & X86_EFL_TF)
9746 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9747 {
9748 /* Bit 14 is PendingDebug.BS. */
9749 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9750 }
9751 }
9752
9753 /* VMCS link pointer. */
9754 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9755 AssertRCBreak(rc);
9756 if (u64Val != UINT64_C(0xffffffffffffffff))
9757 {
9758 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9759 /** @todo Bits beyond the processor's physical-address width MBZ. */
9760 /** @todo 32-bit located in memory referenced by value of this field (as a
9761 * physical address) must contain the processor's VMCS revision ID. */
9762 /** @todo SMM checks. */
9763 }
9764
9765 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9766 * not using Nested Paging? */
9767 if ( pVM->hm.s.fNestedPaging
9768 && !fLongModeGuest
9769 && CPUMIsGuestInPAEModeEx(pCtx))
9770 {
9771 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9772 AssertRCBreak(rc);
9773 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9774
9775 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9776 AssertRCBreak(rc);
9777 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9778
9779 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9780 AssertRCBreak(rc);
9781 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9782
9783 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9784 AssertRCBreak(rc);
9785 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9786 }
9787
9788 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9789 if (uError == VMX_IGS_ERROR)
9790 uError = VMX_IGS_REASON_NOT_FOUND;
9791 } while (0);
9792
9793 pVCpu->hm.s.u32HMError = uError;
9794 return uError;
9795
9796#undef HMVMX_ERROR_BREAK
9797#undef HMVMX_CHECK_BREAK
9798}
9799
9800/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9801/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9802/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9803
9804/** @name VM-exit handlers.
9805 * @{
9806 */
9807
9808/**
9809 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9810 */
9811HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9812{
9813 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9814 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9815 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9816 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
9817 return VINF_SUCCESS;
9818 return VINF_EM_RAW_INTERRUPT;
9819}
9820
9821
9822/**
9823 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9824 */
9825HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9826{
9827 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9828 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9829
9830 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9831 AssertRCReturn(rc, rc);
9832
9833 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9834 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9835 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9836 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9837
9838 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9839 {
9840 /*
9841 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9842 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9843 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9844 *
9845 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9846 */
9847 VMXDispatchHostNmi();
9848 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9849 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9850 return VINF_SUCCESS;
9851 }
9852
9853 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9854 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9855 if (RT_UNLIKELY(rc != VINF_SUCCESS))
9856 {
9857 if (rc == VINF_HM_DOUBLE_FAULT)
9858 rc = VINF_SUCCESS;
9859 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9860 return rc;
9861 }
9862
9863 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9864 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9865 switch (uIntType)
9866 {
9867 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9868 Assert(uVector == X86_XCPT_DB);
9869 /* no break */
9870 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9871 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9872 /* no break */
9873 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9874 {
9875 switch (uVector)
9876 {
9877 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9878 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9879 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9880 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9881 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9882 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9883#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9884 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9885 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9886 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9887 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9888 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9889 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9890 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9891 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9892 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9893 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9894 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9895 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9896#endif
9897 default:
9898 {
9899 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9900 AssertRCReturn(rc, rc);
9901
9902 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9903 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9904 {
9905 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9906 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9907 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9908
9909 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9910 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9911 AssertRCReturn(rc, rc);
9912 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9913 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9914 0 /* GCPtrFaultAddress */);
9915 AssertRCReturn(rc, rc);
9916 }
9917 else
9918 {
9919 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9920 pVCpu->hm.s.u32HMError = uVector;
9921 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9922 }
9923 break;
9924 }
9925 }
9926 break;
9927 }
9928
9929 default:
9930 {
9931 pVCpu->hm.s.u32HMError = uExitIntInfo;
9932 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9933 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9934 break;
9935 }
9936 }
9937 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9938 return rc;
9939}
9940
9941
9942/**
9943 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9944 */
9945HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9946{
9947 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9948
9949 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9950 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9951
9952 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9953 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9954 return VINF_SUCCESS;
9955}
9956
9957
9958/**
9959 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9960 */
9961HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9962{
9963 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9964 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
9965 {
9966 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9967 HMVMX_RETURN_UNEXPECTED_EXIT();
9968 }
9969
9970 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
9971
9972 /*
9973 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
9974 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
9975 */
9976 uint32_t uIntrState = 0;
9977 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9978 AssertRCReturn(rc, rc);
9979
9980 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
9981 if ( fBlockSti
9982 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
9983 {
9984 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
9985 }
9986
9987 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
9988 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
9989
9990 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9991 return VINF_SUCCESS;
9992}
9993
9994
9995/**
9996 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9997 */
9998HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9999{
10000 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10001 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10002 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10003}
10004
10005
10006/**
10007 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10008 */
10009HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10010{
10011 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10012 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10013 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10014}
10015
10016
10017/**
10018 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10019 */
10020HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10021{
10022 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10023 PVM pVM = pVCpu->CTX_SUFF(pVM);
10024 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10025 if (RT_LIKELY(rc == VINF_SUCCESS))
10026 {
10027 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10028 Assert(pVmxTransient->cbInstr == 2);
10029 }
10030 else
10031 {
10032 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10033 rc = VERR_EM_INTERPRETER;
10034 }
10035 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10036 return rc;
10037}
10038
10039
10040/**
10041 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10042 */
10043HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10044{
10045 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10046 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10047 AssertRCReturn(rc, rc);
10048
10049 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10050 return VINF_EM_RAW_EMULATE_INSTR;
10051
10052 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10053 HMVMX_RETURN_UNEXPECTED_EXIT();
10054}
10055
10056
10057/**
10058 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10059 */
10060HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10061{
10062 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10063 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10064 AssertRCReturn(rc, rc);
10065
10066 PVM pVM = pVCpu->CTX_SUFF(pVM);
10067 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10068 if (RT_LIKELY(rc == VINF_SUCCESS))
10069 {
10070 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10071 Assert(pVmxTransient->cbInstr == 2);
10072 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10073 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10074 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10075 }
10076 else
10077 rc = VERR_EM_INTERPRETER;
10078 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10079 return rc;
10080}
10081
10082
10083/**
10084 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10085 */
10086HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10087{
10088 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10089 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10090 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10091 AssertRCReturn(rc, rc);
10092
10093 PVM pVM = pVCpu->CTX_SUFF(pVM);
10094 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10095 if (RT_LIKELY(rc == VINF_SUCCESS))
10096 {
10097 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10098 Assert(pVmxTransient->cbInstr == 3);
10099 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10100 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10101 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10102 }
10103 else
10104 {
10105 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10106 rc = VERR_EM_INTERPRETER;
10107 }
10108 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10109 return rc;
10110}
10111
10112
10113/**
10114 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10115 */
10116HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10117{
10118 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10119 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10120 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10121 AssertRCReturn(rc, rc);
10122
10123 PVM pVM = pVCpu->CTX_SUFF(pVM);
10124 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10125 if (RT_LIKELY(rc == VINF_SUCCESS))
10126 {
10127 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10128 Assert(pVmxTransient->cbInstr == 2);
10129 }
10130 else
10131 {
10132 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10133 rc = VERR_EM_INTERPRETER;
10134 }
10135 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10136 return rc;
10137}
10138
10139
10140/**
10141 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10142 */
10143HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10144{
10145 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10146 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10147
10148 if (pVCpu->hm.s.fHypercallsEnabled)
10149 {
10150#if 0
10151 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10152 AssertRCReturn(rc, rc);
10153#else
10154 /* Aggressive state sync. for now. */
10155 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10156 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
10157#endif
10158 AssertRCReturn(rc, rc);
10159
10160 rc = GIMHypercall(pVCpu, pMixedCtx);
10161 if (RT_SUCCESS(rc))
10162 {
10163 /* If the hypercall changes anything other than guest general-purpose registers,
10164 we would need to reload the guest changed bits here before VM-reentry. */
10165 hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10166 return VINF_SUCCESS;
10167 }
10168 }
10169
10170 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10171 return VINF_SUCCESS;
10172}
10173
10174
10175/**
10176 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10177 */
10178HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10179{
10180 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10181 PVM pVM = pVCpu->CTX_SUFF(pVM);
10182 Assert(!pVM->hm.s.fNestedPaging);
10183
10184 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10185 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10186 AssertRCReturn(rc, rc);
10187
10188 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10189 rc = VBOXSTRICTRC_VAL(rc2);
10190 if (RT_LIKELY(rc == VINF_SUCCESS))
10191 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10192 else
10193 {
10194 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10195 pVmxTransient->uExitQualification, rc));
10196 }
10197 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10198 return rc;
10199}
10200
10201
10202/**
10203 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10204 */
10205HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10206{
10207 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10208 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10209 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10210 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10211 AssertRCReturn(rc, rc);
10212
10213 PVM pVM = pVCpu->CTX_SUFF(pVM);
10214 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10215 if (RT_LIKELY(rc == VINF_SUCCESS))
10216 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10217 else
10218 {
10219 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10220 rc = VERR_EM_INTERPRETER;
10221 }
10222 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10223 return rc;
10224}
10225
10226
10227/**
10228 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10229 */
10230HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10231{
10232 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10233 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10234 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10235 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10236 AssertRCReturn(rc, rc);
10237
10238 PVM pVM = pVCpu->CTX_SUFF(pVM);
10239 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10240 rc = VBOXSTRICTRC_VAL(rc2);
10241 if (RT_LIKELY( rc == VINF_SUCCESS
10242 || rc == VINF_EM_HALT))
10243 {
10244 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10245 AssertRCReturn(rc3, rc3);
10246
10247 if ( rc == VINF_EM_HALT
10248 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10249 {
10250 rc = VINF_SUCCESS;
10251 }
10252 }
10253 else
10254 {
10255 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10256 rc = VERR_EM_INTERPRETER;
10257 }
10258 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10259 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10260 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10261 return rc;
10262}
10263
10264
10265/**
10266 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10267 */
10268HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10269{
10270 /*
10271 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10272 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10273 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10274 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10275 */
10276 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10277 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10278 HMVMX_RETURN_UNEXPECTED_EXIT();
10279}
10280
10281
10282/**
10283 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10284 */
10285HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10286{
10287 /*
10288 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10289 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10290 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10291 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10292 */
10293 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10294 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10295 HMVMX_RETURN_UNEXPECTED_EXIT();
10296}
10297
10298
10299/**
10300 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10301 */
10302HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10303{
10304 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10305 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10306 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10307 HMVMX_RETURN_UNEXPECTED_EXIT();
10308}
10309
10310
10311/**
10312 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10313 */
10314HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10315{
10316 /*
10317 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10318 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10319 * See Intel spec. 25.3 "Other Causes of VM-exits".
10320 */
10321 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10322 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10323 HMVMX_RETURN_UNEXPECTED_EXIT();
10324}
10325
10326
10327/**
10328 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10329 * VM-exit.
10330 */
10331HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10332{
10333 /*
10334 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10335 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10336 *
10337 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10338 * See Intel spec. "23.8 Restrictions on VMX operation".
10339 */
10340 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10341 return VINF_SUCCESS;
10342}
10343
10344
10345/**
10346 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10347 * VM-exit.
10348 */
10349HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10350{
10351 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10352 return VINF_EM_RESET;
10353}
10354
10355
10356/**
10357 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10358 */
10359HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10360{
10361 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10362 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10363 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10364 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10365 AssertRCReturn(rc, rc);
10366
10367 pMixedCtx->rip++;
10368 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10369 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10370 rc = VINF_SUCCESS;
10371 else
10372 rc = VINF_EM_HALT;
10373
10374 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10375 if (rc != VINF_SUCCESS)
10376 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
10377 return rc;
10378}
10379
10380
10381/**
10382 * VM-exit handler for instructions that result in a #UD exception delivered to
10383 * the guest.
10384 */
10385HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10386{
10387 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10388 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10389 return VINF_SUCCESS;
10390}
10391
10392
10393/**
10394 * VM-exit handler for expiry of the VMX preemption timer.
10395 */
10396HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10397{
10398 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10399
10400 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10401 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10402
10403 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10404 PVM pVM = pVCpu->CTX_SUFF(pVM);
10405 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10406 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10407 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10408}
10409
10410
10411/**
10412 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10413 */
10414HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10415{
10416 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10417
10418 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10419 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
10420 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10421 AssertRCReturn(rc, rc);
10422
10423 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
10424 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
10425
10426 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
10427
10428 return VBOXSTRICTRC_TODO(rcStrict);
10429}
10430
10431
10432/**
10433 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10434 */
10435HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10436{
10437 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10438
10439 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
10440 /** @todo implement EMInterpretInvpcid() */
10441 return VERR_EM_INTERPRETER;
10442}
10443
10444
10445/**
10446 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10447 * Error VM-exit.
10448 */
10449HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10450{
10451 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10452 AssertRCReturn(rc, rc);
10453
10454 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10455 AssertRCReturn(rc, rc);
10456
10457 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10458 NOREF(uInvalidReason);
10459
10460#ifdef VBOX_STRICT
10461 uint32_t uIntrState;
10462 RTHCUINTREG uHCReg;
10463 uint64_t u64Val;
10464 uint32_t u32Val;
10465
10466 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10467 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10468 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10469 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10470 AssertRCReturn(rc, rc);
10471
10472 Log4(("uInvalidReason %u\n", uInvalidReason));
10473 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10474 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10475 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10476 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10477
10478 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10479 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10480 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10481 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10482 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10483 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10484 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10485 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10486 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10487 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10488 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10489 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10490#else
10491 NOREF(pVmxTransient);
10492#endif
10493
10494 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10495 return VERR_VMX_INVALID_GUEST_STATE;
10496}
10497
10498
10499/**
10500 * VM-exit handler for VM-entry failure due to an MSR-load
10501 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10502 */
10503HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10504{
10505 NOREF(pVmxTransient);
10506 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10507 HMVMX_RETURN_UNEXPECTED_EXIT();
10508}
10509
10510
10511/**
10512 * VM-exit handler for VM-entry failure due to a machine-check event
10513 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10514 */
10515HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10516{
10517 NOREF(pVmxTransient);
10518 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10519 HMVMX_RETURN_UNEXPECTED_EXIT();
10520}
10521
10522
10523/**
10524 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10525 * theory.
10526 */
10527HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10528{
10529 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10530 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10531 return VERR_VMX_UNDEFINED_EXIT_CODE;
10532}
10533
10534
10535/**
10536 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10537 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10538 * Conditional VM-exit.
10539 */
10540HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10541{
10542 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10543
10544 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10545 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10546 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10547 return VERR_EM_INTERPRETER;
10548 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10549 HMVMX_RETURN_UNEXPECTED_EXIT();
10550}
10551
10552
10553/**
10554 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10555 */
10556HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10557{
10558 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10559
10560 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10561 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10562 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10563 return VERR_EM_INTERPRETER;
10564 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10565 HMVMX_RETURN_UNEXPECTED_EXIT();
10566}
10567
10568
10569/**
10570 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10571 */
10572HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10573{
10574 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10575
10576 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10577 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10578 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10579 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10580 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10581 {
10582 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10583 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10584 }
10585 AssertRCReturn(rc, rc);
10586 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10587
10588#ifdef VBOX_STRICT
10589 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10590 {
10591 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10592 && pMixedCtx->ecx != MSR_K6_EFER)
10593 {
10594 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10595 pMixedCtx->ecx));
10596 HMVMX_RETURN_UNEXPECTED_EXIT();
10597 }
10598# if HC_ARCH_BITS == 64
10599 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10600 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10601 {
10602 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10603 HMVMX_RETURN_UNEXPECTED_EXIT();
10604 }
10605# endif
10606 }
10607#endif
10608
10609 PVM pVM = pVCpu->CTX_SUFF(pVM);
10610 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10611 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10612 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10613 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10614 if (RT_LIKELY(rc == VINF_SUCCESS))
10615 {
10616 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10617 Assert(pVmxTransient->cbInstr == 2);
10618 }
10619 return rc;
10620}
10621
10622
10623/**
10624 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10625 */
10626HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10627{
10628 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10629 PVM pVM = pVCpu->CTX_SUFF(pVM);
10630 int rc = VINF_SUCCESS;
10631
10632 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10633 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10634 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10635 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10636 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10637 {
10638 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10639 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10640 }
10641 AssertRCReturn(rc, rc);
10642 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10643
10644 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10645 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10646 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10647
10648 if (RT_LIKELY(rc == VINF_SUCCESS))
10649 {
10650 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10651
10652 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10653 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10654 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10655 {
10656 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10657 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10658 EMInterpretWrmsr() changes it. */
10659 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10660 }
10661 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10662 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10663 else if (pMixedCtx->ecx == MSR_K6_EFER)
10664 {
10665 /*
10666 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10667 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10668 * the other bits as well, SCE and NXE. See @bugref{7368}.
10669 */
10670 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10671 }
10672
10673 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10674 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10675 {
10676 switch (pMixedCtx->ecx)
10677 {
10678 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10679 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10680 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10681 case MSR_K8_FS_BASE: /* no break */
10682 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10683 case MSR_K6_EFER: /* already handled above */ break;
10684 default:
10685 {
10686 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10687 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10688#if HC_ARCH_BITS == 64
10689 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10690 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10691#endif
10692 break;
10693 }
10694 }
10695 }
10696#ifdef VBOX_STRICT
10697 else
10698 {
10699 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10700 switch (pMixedCtx->ecx)
10701 {
10702 case MSR_IA32_SYSENTER_CS:
10703 case MSR_IA32_SYSENTER_EIP:
10704 case MSR_IA32_SYSENTER_ESP:
10705 case MSR_K8_FS_BASE:
10706 case MSR_K8_GS_BASE:
10707 {
10708 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10709 HMVMX_RETURN_UNEXPECTED_EXIT();
10710 }
10711
10712 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10713 default:
10714 {
10715 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10716 {
10717 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10718 if (pMixedCtx->ecx != MSR_K6_EFER)
10719 {
10720 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10721 pMixedCtx->ecx));
10722 HMVMX_RETURN_UNEXPECTED_EXIT();
10723 }
10724 }
10725
10726#if HC_ARCH_BITS == 64
10727 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10728 {
10729 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10730 HMVMX_RETURN_UNEXPECTED_EXIT();
10731 }
10732#endif
10733 break;
10734 }
10735 }
10736 }
10737#endif /* VBOX_STRICT */
10738 }
10739 return rc;
10740}
10741
10742
10743/**
10744 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10745 */
10746HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10747{
10748 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10749
10750 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10751 return VINF_EM_RAW_INTERRUPT;
10752}
10753
10754
10755/**
10756 * VM-exit handler for when the TPR value is lowered below the specified
10757 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10758 */
10759HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10760{
10761 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10762 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10763
10764 /*
10765 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10766 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10767 * resume guest execution.
10768 */
10769 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10770 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10771 return VINF_SUCCESS;
10772}
10773
10774
10775/**
10776 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10777 * VM-exit.
10778 *
10779 * @retval VINF_SUCCESS when guest execution can continue.
10780 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10781 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10782 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10783 * interpreter.
10784 */
10785HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10786{
10787 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10788 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10789 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10790 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10791 AssertRCReturn(rc, rc);
10792
10793 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10794 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10795 PVM pVM = pVCpu->CTX_SUFF(pVM);
10796 VBOXSTRICTRC rcStrict;
10797 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
10798 switch (uAccessType)
10799 {
10800 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10801 {
10802 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10803 AssertRCReturn(rc, rc);
10804
10805 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
10806 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10807 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10808 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
10809 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10810 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10811 {
10812 case 0: /* CR0 */
10813 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10814 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
10815 break;
10816 case 2: /* CR2 */
10817 /* Nothing to do here, CR2 it's not part of the VMCS. */
10818 break;
10819 case 3: /* CR3 */
10820 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10821 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10822 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
10823 break;
10824 case 4: /* CR4 */
10825 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10826 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
10827 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
10828 break;
10829 case 8: /* CR8 */
10830 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10831 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
10832 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10833 break;
10834 default:
10835 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10836 break;
10837 }
10838
10839 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10840 break;
10841 }
10842
10843 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10844 {
10845 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10846 AssertRCReturn(rc, rc);
10847
10848 Assert( !pVM->hm.s.fNestedPaging
10849 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10850 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10851
10852 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10853 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10854 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10855
10856 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
10857 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10858 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10859 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10860 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10861 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10862 VBOXSTRICTRC_VAL(rcStrict)));
10863 break;
10864 }
10865
10866 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10867 {
10868 AssertRCReturn(rc, rc);
10869 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
10870 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10871 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10872 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10873 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
10874 break;
10875 }
10876
10877 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10878 {
10879 AssertRCReturn(rc, rc);
10880 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
10881 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10882 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
10883 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10884 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10885 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
10886 break;
10887 }
10888
10889 default:
10890 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
10891 VERR_VMX_UNEXPECTED_EXCEPTION);
10892 }
10893
10894 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
10895 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10896 return VBOXSTRICTRC_TODO(rcStrict);
10897}
10898
10899
10900/**
10901 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10902 * VM-exit.
10903 */
10904HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10905{
10906 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10907 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10908
10909 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10910 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10911 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10912 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10913 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10914 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10915 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10916 AssertRCReturn(rc2, rc2);
10917
10918 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10919 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10920 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10921 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10922 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10923 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10924 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10925 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10926
10927 /* I/O operation lookup arrays. */
10928 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10929 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10930
10931 VBOXSTRICTRC rcStrict;
10932 uint32_t const cbValue = s_aIOSizes[uIOWidth];
10933 uint32_t const cbInstr = pVmxTransient->cbInstr;
10934 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10935 PVM pVM = pVCpu->CTX_SUFF(pVM);
10936 if (fIOString)
10937 {
10938#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
10939 See @bugref{5752#c158}. Should work now. */
10940 /*
10941 * INS/OUTS - I/O String instruction.
10942 *
10943 * Use instruction-information if available, otherwise fall back on
10944 * interpreting the instruction.
10945 */
10946 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
10947 fIOWrite ? 'w' : 'r'));
10948 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
10949 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10950 {
10951 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10952 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10953 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10954 AssertRCReturn(rc2, rc2);
10955 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
10956 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10957 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10958 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10959 if (fIOWrite)
10960 {
10961 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10962 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10963 }
10964 else
10965 {
10966 /*
10967 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10968 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10969 * See Intel Instruction spec. for "INS".
10970 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10971 */
10972 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10973 }
10974 }
10975 else
10976 {
10977 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10978 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10979 AssertRCReturn(rc2, rc2);
10980 rcStrict = IEMExecOne(pVCpu);
10981 }
10982 /** @todo IEM needs to be setting these flags somehow. */
10983 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10984 fUpdateRipAlready = true;
10985#else
10986 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10987 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10988 if (RT_SUCCESS(rcStrict))
10989 {
10990 if (fIOWrite)
10991 {
10992 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10993 (DISCPUMODE)pDis->uAddrMode, cbValue);
10994 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10995 }
10996 else
10997 {
10998 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10999 (DISCPUMODE)pDis->uAddrMode, cbValue);
11000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11001 }
11002 }
11003 else
11004 {
11005 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
11006 pMixedCtx->rip));
11007 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11008 }
11009#endif
11010 }
11011 else
11012 {
11013 /*
11014 * IN/OUT - I/O instruction.
11015 */
11016 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11017 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11018 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11019 if (fIOWrite)
11020 {
11021 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11022 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11023 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11024 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11025 }
11026 else
11027 {
11028 uint32_t u32Result = 0;
11029 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11030 if (IOM_SUCCESS(rcStrict))
11031 {
11032 /* Save result of I/O IN instr. in AL/AX/EAX. */
11033 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11034 }
11035 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11036 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11037 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11038 }
11039 }
11040
11041 if (IOM_SUCCESS(rcStrict))
11042 {
11043 if (!fUpdateRipAlready)
11044 {
11045 pMixedCtx->rip += cbInstr;
11046 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11047 }
11048
11049 /*
11050 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11051 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11052 */
11053 if (fIOString)
11054 {
11055 /** @todo Single-step for INS/OUTS with REP prefix? */
11056 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11057 }
11058 else if (fStepping)
11059 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11060
11061 /*
11062 * If any I/O breakpoints are armed, we need to check if one triggered
11063 * and take appropriate action.
11064 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11065 */
11066 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11067 AssertRCReturn(rc2, rc2);
11068
11069 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11070 * execution engines about whether hyper BPs and such are pending. */
11071 uint32_t const uDr7 = pMixedCtx->dr[7];
11072 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11073 && X86_DR7_ANY_RW_IO(uDr7)
11074 && (pMixedCtx->cr4 & X86_CR4_DE))
11075 || DBGFBpIsHwIoArmed(pVM)))
11076 {
11077 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11078
11079 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11080 VMMRZCallRing3Disable(pVCpu);
11081 HM_DISABLE_PREEMPT();
11082
11083 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11084
11085 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11086 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11087 {
11088 /* Raise #DB. */
11089 if (fIsGuestDbgActive)
11090 ASMSetDR6(pMixedCtx->dr[6]);
11091 if (pMixedCtx->dr[7] != uDr7)
11092 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11093
11094 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11095 }
11096 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11097 else if ( rcStrict2 != VINF_SUCCESS
11098 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11099 rcStrict = rcStrict2;
11100
11101 HM_RESTORE_PREEMPT();
11102 VMMRZCallRing3Enable(pVCpu);
11103 }
11104 }
11105
11106#ifdef DEBUG
11107 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11108 Assert(!fIOWrite);
11109 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11110 Assert(fIOWrite);
11111 else
11112 {
11113 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11114 * statuses, that the VMM device and some others may return. See
11115 * IOM_SUCCESS() for guidance. */
11116 AssertMsg( RT_FAILURE(rcStrict)
11117 || rcStrict == VINF_SUCCESS
11118 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11119 || rcStrict == VINF_EM_DBG_BREAKPOINT
11120 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11121 || rcStrict == VINF_EM_RAW_TO_R3
11122 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11123 }
11124#endif
11125
11126 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11127 return VBOXSTRICTRC_TODO(rcStrict);
11128}
11129
11130
11131/**
11132 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11133 * VM-exit.
11134 */
11135HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11136{
11137 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11138
11139 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11140 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11141 AssertRCReturn(rc, rc);
11142 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11143 {
11144 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11145 AssertRCReturn(rc, rc);
11146 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11147 {
11148 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11149
11150 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11151 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11152
11153 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11154 Assert(!pVCpu->hm.s.Event.fPending);
11155 pVCpu->hm.s.Event.fPending = true;
11156 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11157 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11158 AssertRCReturn(rc, rc);
11159 if (fErrorCodeValid)
11160 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11161 else
11162 pVCpu->hm.s.Event.u32ErrCode = 0;
11163 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11164 && uVector == X86_XCPT_PF)
11165 {
11166 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11167 }
11168
11169 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11170 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11171 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11172 }
11173 }
11174
11175 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11176 * emulation. */
11177 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11178 return VERR_EM_INTERPRETER;
11179}
11180
11181
11182/**
11183 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11184 */
11185HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11186{
11187 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11188 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11189 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11190 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11191 AssertRCReturn(rc, rc);
11192 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11193 return VINF_EM_DBG_STEPPED;
11194}
11195
11196
11197/**
11198 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11199 */
11200HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11201{
11202 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11203
11204 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11205 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11206 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11207 {
11208 if (rc == VINF_HM_DOUBLE_FAULT)
11209 rc = VINF_SUCCESS;
11210 return rc;
11211 }
11212
11213#if 0
11214 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11215 * just sync the whole thing. */
11216 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11217#else
11218 /* Aggressive state sync. for now. */
11219 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11220 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11221 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11222#endif
11223 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11224 AssertRCReturn(rc, rc);
11225
11226 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11227 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11228 switch (uAccessType)
11229 {
11230 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11231 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11232 {
11233 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11234 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11235 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11236
11237 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11238 GCPhys &= PAGE_BASE_GC_MASK;
11239 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11240 PVM pVM = pVCpu->CTX_SUFF(pVM);
11241 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11242 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11243
11244 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11245 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11246 CPUMCTX2CORE(pMixedCtx), GCPhys);
11247 rc = VBOXSTRICTRC_VAL(rc2);
11248 Log4(("ApicAccess rc=%d\n", rc));
11249 if ( rc == VINF_SUCCESS
11250 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11251 || rc == VERR_PAGE_NOT_PRESENT)
11252 {
11253 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11254 | HM_CHANGED_GUEST_RSP
11255 | HM_CHANGED_GUEST_RFLAGS
11256 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11257 rc = VINF_SUCCESS;
11258 }
11259 break;
11260 }
11261
11262 default:
11263 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11264 rc = VINF_EM_RAW_EMULATE_INSTR;
11265 break;
11266 }
11267
11268 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11269 if (rc != VINF_SUCCESS)
11270 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
11271 return rc;
11272}
11273
11274
11275/**
11276 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11277 * VM-exit.
11278 */
11279HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11280{
11281 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11282
11283 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11284 if (pVmxTransient->fWasGuestDebugStateActive)
11285 {
11286 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11287 HMVMX_RETURN_UNEXPECTED_EXIT();
11288 }
11289
11290 int rc = VERR_INTERNAL_ERROR_5;
11291 if ( !DBGFIsStepping(pVCpu)
11292 && !pVCpu->hm.s.fSingleInstruction
11293 && !pVmxTransient->fWasHyperDebugStateActive)
11294 {
11295 /* Don't intercept MOV DRx and #DB any more. */
11296 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11297 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11298 AssertRCReturn(rc, rc);
11299
11300 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11301 {
11302#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11303 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11304 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
11305#endif
11306 }
11307
11308 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11309 VMMRZCallRing3Disable(pVCpu);
11310 HM_DISABLE_PREEMPT();
11311
11312 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11313 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11314 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11315
11316 HM_RESTORE_PREEMPT();
11317 VMMRZCallRing3Enable(pVCpu);
11318
11319#ifdef VBOX_WITH_STATISTICS
11320 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11321 AssertRCReturn(rc, rc);
11322 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11323 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11324 else
11325 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11326#endif
11327 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11328 return VINF_SUCCESS;
11329 }
11330
11331 /*
11332 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11333 * Update the segment registers and DR7 from the CPU.
11334 */
11335 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11336 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11337 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11338 AssertRCReturn(rc, rc);
11339 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11340
11341 PVM pVM = pVCpu->CTX_SUFF(pVM);
11342 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11343 {
11344 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11345 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11346 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11347 if (RT_SUCCESS(rc))
11348 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11349 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11350 }
11351 else
11352 {
11353 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11354 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11355 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11357 }
11358
11359 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11360 if (RT_SUCCESS(rc))
11361 {
11362 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11363 AssertRCReturn(rc2, rc2);
11364 }
11365 return rc;
11366}
11367
11368
11369/**
11370 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11371 * Conditional VM-exit.
11372 */
11373HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11374{
11375 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11376 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11377
11378 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11379 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11380 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11381 {
11382 if (rc == VINF_HM_DOUBLE_FAULT)
11383 rc = VINF_SUCCESS;
11384 return rc;
11385 }
11386
11387 RTGCPHYS GCPhys = 0;
11388 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11389
11390#if 0
11391 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11392#else
11393 /* Aggressive state sync. for now. */
11394 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11395 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11396 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11397#endif
11398 AssertRCReturn(rc, rc);
11399
11400 /*
11401 * If we succeed, resume guest execution.
11402 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11403 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11404 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11405 * weird case. See @bugref{6043}.
11406 */
11407 PVM pVM = pVCpu->CTX_SUFF(pVM);
11408 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11409 rc = VBOXSTRICTRC_VAL(rc2);
11410 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11411 if ( rc == VINF_SUCCESS
11412 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11413 || rc == VERR_PAGE_NOT_PRESENT)
11414 {
11415 /* Successfully handled MMIO operation. */
11416 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11417 | HM_CHANGED_GUEST_RSP
11418 | HM_CHANGED_GUEST_RFLAGS
11419 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11420 rc = VINF_SUCCESS;
11421 }
11422 return rc;
11423}
11424
11425
11426/**
11427 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11428 * VM-exit.
11429 */
11430HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11431{
11432 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11433 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11434
11435 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11436 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11437 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11438 {
11439 if (rc == VINF_HM_DOUBLE_FAULT)
11440 rc = VINF_SUCCESS;
11441 return rc;
11442 }
11443
11444 RTGCPHYS GCPhys = 0;
11445 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11446 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11447#if 0
11448 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11449#else
11450 /* Aggressive state sync. for now. */
11451 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11452 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11453 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11454#endif
11455 AssertRCReturn(rc, rc);
11456
11457 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11458 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11459
11460 RTGCUINT uErrorCode = 0;
11461 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11462 uErrorCode |= X86_TRAP_PF_ID;
11463 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11464 uErrorCode |= X86_TRAP_PF_RW;
11465 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11466 uErrorCode |= X86_TRAP_PF_P;
11467
11468 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11469
11470 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11471 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11472
11473 /* Handle the pagefault trap for the nested shadow table. */
11474 PVM pVM = pVCpu->CTX_SUFF(pVM);
11475 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11476 TRPMResetTrap(pVCpu);
11477
11478 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11479 if ( rc == VINF_SUCCESS
11480 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11481 || rc == VERR_PAGE_NOT_PRESENT)
11482 {
11483 /* Successfully synced our nested page tables. */
11484 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11485 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11486 | HM_CHANGED_GUEST_RSP
11487 | HM_CHANGED_GUEST_RFLAGS);
11488 return VINF_SUCCESS;
11489 }
11490
11491 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11492 return rc;
11493}
11494
11495/** @} */
11496
11497/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11498/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11499/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11500
11501/** @name VM-exit exception handlers.
11502 * @{
11503 */
11504
11505/**
11506 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11507 */
11508static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11509{
11510 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11511 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11512
11513 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11514 AssertRCReturn(rc, rc);
11515
11516 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11517 {
11518 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11519 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11520
11521 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11522 * provides VM-exit instruction length. If this causes problem later,
11523 * disassemble the instruction like it's done on AMD-V. */
11524 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11525 AssertRCReturn(rc2, rc2);
11526 return rc;
11527 }
11528
11529 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11530 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11531 return rc;
11532}
11533
11534
11535/**
11536 * VM-exit exception handler for #BP (Breakpoint exception).
11537 */
11538static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11539{
11540 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11541 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11542
11543 /** @todo Try optimize this by not saving the entire guest state unless
11544 * really needed. */
11545 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11546 AssertRCReturn(rc, rc);
11547
11548 PVM pVM = pVCpu->CTX_SUFF(pVM);
11549 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11550 if (rc == VINF_EM_RAW_GUEST_TRAP)
11551 {
11552 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11553 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11554 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11555 AssertRCReturn(rc, rc);
11556
11557 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11558 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11559 }
11560
11561 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11562 return rc;
11563}
11564
11565
11566/**
11567 * VM-exit exception handler for #DB (Debug exception).
11568 */
11569static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11570{
11571 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11572 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11573 Log6(("XcptDB\n"));
11574
11575 /*
11576 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11577 * for processing.
11578 */
11579 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11580 AssertRCReturn(rc, rc);
11581
11582 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11583 uint64_t uDR6 = X86_DR6_INIT_VAL;
11584 uDR6 |= ( pVmxTransient->uExitQualification
11585 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11586
11587 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11588 if (rc == VINF_EM_RAW_GUEST_TRAP)
11589 {
11590 /*
11591 * The exception was for the guest. Update DR6, DR7.GD and
11592 * IA32_DEBUGCTL.LBR before forwarding it.
11593 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11594 */
11595 VMMRZCallRing3Disable(pVCpu);
11596 HM_DISABLE_PREEMPT();
11597
11598 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11599 pMixedCtx->dr[6] |= uDR6;
11600 if (CPUMIsGuestDebugStateActive(pVCpu))
11601 ASMSetDR6(pMixedCtx->dr[6]);
11602
11603 HM_RESTORE_PREEMPT();
11604 VMMRZCallRing3Enable(pVCpu);
11605
11606 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11607 AssertRCReturn(rc, rc);
11608
11609 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11610 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11611
11612 /* Paranoia. */
11613 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11614 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11615
11616 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11617 AssertRCReturn(rc, rc);
11618
11619 /*
11620 * Raise #DB in the guest.
11621 *
11622 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11623 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11624 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11625 *
11626 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11627 */
11628 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11629 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11630 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11631 AssertRCReturn(rc, rc);
11632 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11633 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11634 return VINF_SUCCESS;
11635 }
11636
11637 /*
11638 * Not a guest trap, must be a hypervisor related debug event then.
11639 * Update DR6 in case someone is interested in it.
11640 */
11641 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11642 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11643 CPUMSetHyperDR6(pVCpu, uDR6);
11644
11645 return rc;
11646}
11647
11648
11649/**
11650 * VM-exit exception handler for #NM (Device-not-available exception: floating
11651 * point exception).
11652 */
11653static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11654{
11655 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11656
11657 /* We require CR0 and EFER. EFER is always up-to-date. */
11658 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11659 AssertRCReturn(rc, rc);
11660
11661 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11662 VMMRZCallRing3Disable(pVCpu);
11663 HM_DISABLE_PREEMPT();
11664
11665 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11666 if (pVmxTransient->fWasGuestFPUStateActive)
11667 {
11668 rc = VINF_EM_RAW_GUEST_TRAP;
11669 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11670 }
11671 else
11672 {
11673#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11674 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11675#endif
11676 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11677 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11678 }
11679
11680 HM_RESTORE_PREEMPT();
11681 VMMRZCallRing3Enable(pVCpu);
11682
11683 if (rc == VINF_SUCCESS)
11684 {
11685 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11686 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11687 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11688 pVCpu->hm.s.fPreloadGuestFpu = true;
11689 }
11690 else
11691 {
11692 /* Forward #NM to the guest. */
11693 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11694 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11695 AssertRCReturn(rc, rc);
11696 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11697 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11698 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11699 }
11700
11701 return VINF_SUCCESS;
11702}
11703
11704
11705/**
11706 * VM-exit exception handler for #GP (General-protection exception).
11707 *
11708 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11709 */
11710static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11711{
11712 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11713 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11714
11715 int rc = VERR_INTERNAL_ERROR_5;
11716 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11717 {
11718#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11719 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11720 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11721 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11722 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11723 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11724 AssertRCReturn(rc, rc);
11725 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11726 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11727 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11728 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11729 return rc;
11730#else
11731 /* We don't intercept #GP. */
11732 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11733 NOREF(pVmxTransient);
11734 return VERR_VMX_UNEXPECTED_EXCEPTION;
11735#endif
11736 }
11737
11738 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11739 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11740
11741 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11742 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11743 AssertRCReturn(rc, rc);
11744
11745 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11746 uint32_t cbOp = 0;
11747 PVM pVM = pVCpu->CTX_SUFF(pVM);
11748 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11749 if (RT_SUCCESS(rc))
11750 {
11751 rc = VINF_SUCCESS;
11752 Assert(cbOp == pDis->cbInstr);
11753 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11754 switch (pDis->pCurInstr->uOpcode)
11755 {
11756 case OP_CLI:
11757 {
11758 pMixedCtx->eflags.Bits.u1IF = 0;
11759 pMixedCtx->eflags.Bits.u1RF = 0;
11760 pMixedCtx->rip += pDis->cbInstr;
11761 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11762 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11763 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11764 break;
11765 }
11766
11767 case OP_STI:
11768 {
11769 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11770 pMixedCtx->eflags.Bits.u1IF = 1;
11771 pMixedCtx->eflags.Bits.u1RF = 0;
11772 pMixedCtx->rip += pDis->cbInstr;
11773 if (!fOldIF)
11774 {
11775 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11776 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11777 }
11778 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11779 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11780 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11781 break;
11782 }
11783
11784 case OP_HLT:
11785 {
11786 rc = VINF_EM_HALT;
11787 pMixedCtx->rip += pDis->cbInstr;
11788 pMixedCtx->eflags.Bits.u1RF = 0;
11789 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11790 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11791 break;
11792 }
11793
11794 case OP_POPF:
11795 {
11796 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11797 uint32_t cbParm;
11798 uint32_t uMask;
11799 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11800 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11801 {
11802 cbParm = 4;
11803 uMask = 0xffffffff;
11804 }
11805 else
11806 {
11807 cbParm = 2;
11808 uMask = 0xffff;
11809 }
11810
11811 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11812 RTGCPTR GCPtrStack = 0;
11813 X86EFLAGS Eflags;
11814 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11815 &GCPtrStack);
11816 if (RT_SUCCESS(rc))
11817 {
11818 Assert(sizeof(Eflags.u32) >= cbParm);
11819 Eflags.u32 = 0;
11820 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
11821 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
11822 }
11823 if (RT_FAILURE(rc))
11824 {
11825 rc = VERR_EM_INTERPRETER;
11826 break;
11827 }
11828 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11829 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
11830 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11831 pMixedCtx->esp += cbParm;
11832 pMixedCtx->esp &= uMask;
11833 pMixedCtx->rip += pDis->cbInstr;
11834 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11835 | HM_CHANGED_GUEST_RSP
11836 | HM_CHANGED_GUEST_RFLAGS);
11837 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11838 if (fStepping)
11839 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11840
11841 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11842 break;
11843 }
11844
11845 case OP_PUSHF:
11846 {
11847 uint32_t cbParm;
11848 uint32_t uMask;
11849 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11850 {
11851 cbParm = 4;
11852 uMask = 0xffffffff;
11853 }
11854 else
11855 {
11856 cbParm = 2;
11857 uMask = 0xffff;
11858 }
11859
11860 /* Get the stack pointer & push the contents of eflags onto the stack. */
11861 RTGCPTR GCPtrStack = 0;
11862 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11863 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11864 if (RT_FAILURE(rc))
11865 {
11866 rc = VERR_EM_INTERPRETER;
11867 break;
11868 }
11869 X86EFLAGS Eflags = pMixedCtx->eflags;
11870 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11871 Eflags.Bits.u1RF = 0;
11872 Eflags.Bits.u1VM = 0;
11873
11874 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
11875 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11876 {
11877 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
11878 rc = VERR_EM_INTERPRETER;
11879 break;
11880 }
11881 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11882 pMixedCtx->esp -= cbParm;
11883 pMixedCtx->esp &= uMask;
11884 pMixedCtx->rip += pDis->cbInstr;
11885 pMixedCtx->eflags.Bits.u1RF = 0;
11886 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11887 | HM_CHANGED_GUEST_RSP
11888 | HM_CHANGED_GUEST_RFLAGS);
11889 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11890 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11891 break;
11892 }
11893
11894 case OP_IRET:
11895 {
11896 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11897 * instruction reference. */
11898 RTGCPTR GCPtrStack = 0;
11899 uint32_t uMask = 0xffff;
11900 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11901 uint16_t aIretFrame[3];
11902 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11903 {
11904 rc = VERR_EM_INTERPRETER;
11905 break;
11906 }
11907 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11908 &GCPtrStack);
11909 if (RT_SUCCESS(rc))
11910 {
11911 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
11912 PGMACCESSORIGIN_HM));
11913 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
11914 }
11915 if (RT_FAILURE(rc))
11916 {
11917 rc = VERR_EM_INTERPRETER;
11918 break;
11919 }
11920 pMixedCtx->eip = 0;
11921 pMixedCtx->ip = aIretFrame[0];
11922 pMixedCtx->cs.Sel = aIretFrame[1];
11923 pMixedCtx->cs.ValidSel = aIretFrame[1];
11924 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11925 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
11926 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11927 pMixedCtx->sp += sizeof(aIretFrame);
11928 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11929 | HM_CHANGED_GUEST_SEGMENT_REGS
11930 | HM_CHANGED_GUEST_RSP
11931 | HM_CHANGED_GUEST_RFLAGS);
11932 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11933 if (fStepping)
11934 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11935 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11936 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11937 break;
11938 }
11939
11940 case OP_INT:
11941 {
11942 uint16_t uVector = pDis->Param1.uValue & 0xff;
11943 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11944 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11945 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11946 break;
11947 }
11948
11949 case OP_INTO:
11950 {
11951 if (pMixedCtx->eflags.Bits.u1OF)
11952 {
11953 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11954 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11955 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11956 }
11957 else
11958 {
11959 pMixedCtx->eflags.Bits.u1RF = 0;
11960 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11961 }
11962 break;
11963 }
11964
11965 default:
11966 {
11967 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
11968 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11969 EMCODETYPE_SUPERVISOR);
11970 rc = VBOXSTRICTRC_VAL(rc2);
11971 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11972 /** @todo We have to set pending-debug exceptions here when the guest is
11973 * single-stepping depending on the instruction that was interpreted. */
11974 Log4(("#GP rc=%Rrc\n", rc));
11975 break;
11976 }
11977 }
11978 }
11979 else
11980 rc = VERR_EM_INTERPRETER;
11981
11982 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11983 ("#GP Unexpected rc=%Rrc\n", rc));
11984 return rc;
11985}
11986
11987
11988#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11989/**
11990 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11991 * the exception reported in the VMX transient structure back into the VM.
11992 *
11993 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11994 * up-to-date.
11995 */
11996static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11997{
11998 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11999
12000 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12001 hmR0VmxCheckExitDueToEventDelivery(). */
12002 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12003 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12004 AssertRCReturn(rc, rc);
12005 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12006
12007#ifdef DEBUG_ramshankar
12008 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12009 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12010 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12011#endif
12012
12013 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12014 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12015 return VINF_SUCCESS;
12016}
12017#endif
12018
12019
12020/**
12021 * VM-exit exception handler for #PF (Page-fault exception).
12022 */
12023static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12024{
12025 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12026 PVM pVM = pVCpu->CTX_SUFF(pVM);
12027 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12028 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12029 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12030 AssertRCReturn(rc, rc);
12031
12032#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12033 if (pVM->hm.s.fNestedPaging)
12034 {
12035 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12036 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12037 {
12038 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12039 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12040 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12041 }
12042 else
12043 {
12044 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12045 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12046 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12047 }
12048 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12049 return rc;
12050 }
12051#else
12052 Assert(!pVM->hm.s.fNestedPaging);
12053 NOREF(pVM);
12054#endif
12055
12056 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12057 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12058 if (pVmxTransient->fVectoringPF)
12059 {
12060 Assert(pVCpu->hm.s.Event.fPending);
12061 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12062 }
12063
12064 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12065 AssertRCReturn(rc, rc);
12066
12067 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12068 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12069
12070 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12071 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12072 (RTGCPTR)pVmxTransient->uExitQualification);
12073
12074 Log4(("#PF: rc=%Rrc\n", rc));
12075 if (rc == VINF_SUCCESS)
12076 {
12077 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12078 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12079 * memory? We don't update the whole state here... */
12080 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12081 | HM_CHANGED_GUEST_RSP
12082 | HM_CHANGED_GUEST_RFLAGS
12083 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12084 TRPMResetTrap(pVCpu);
12085 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12086 return rc;
12087 }
12088
12089 if (rc == VINF_EM_RAW_GUEST_TRAP)
12090 {
12091 if (!pVmxTransient->fVectoringDoublePF)
12092 {
12093 /* It's a guest page fault and needs to be reflected to the guest. */
12094 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12095 TRPMResetTrap(pVCpu);
12096 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12097 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12098 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12099 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12100 }
12101 else
12102 {
12103 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12104 TRPMResetTrap(pVCpu);
12105 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12106 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12107 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12108 }
12109
12110 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12111 return VINF_SUCCESS;
12112 }
12113
12114 TRPMResetTrap(pVCpu);
12115 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12116 return rc;
12117}
12118
12119/** @} */
12120
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette