VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 57854

Last change on this file since 57854 was 57854, checked in by vboxsync, 9 years ago

VMM: unused parameter warning fixes.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 505.5 KB
Line 
1/* $Id: HMVMXR0.cpp 57854 2015-09-22 13:47:01Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#include "HMInternal.h"
38#include <VBox/vmm/vm.h>
39#include "HMVMXR0.h"
40#include "dtrace/VBoxVMM.h"
41
42#ifdef DEBUG_ramshankar
43# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
44# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_CHECK_GUEST_STATE
47# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
48# define HMVMX_ALWAYS_TRAP_PF
49# define HMVMX_ALWAYS_SWAP_FPU_STATE
50# define HMVMX_ALWAYS_FLUSH_TLB
51# define HMVMX_ALWAYS_SWAP_EFER
52#endif
53
54
55/*********************************************************************************************************************************
56* Defined Constants And Macros *
57*********************************************************************************************************************************/
58/** Use the function table. */
59#define HMVMX_USE_FUNCTION_TABLE
60
61/** Determine which tagged-TLB flush handler to use. */
62#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
63#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
64#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
65#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
66
67/** @name Updated-guest-state flags.
68 * @{ */
69#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
70#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
71#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
72#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
73#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
74#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
75#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
76#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
77#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
78#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
79#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
80#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
81#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
82#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
83#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
84#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
85#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
86#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
87#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
88#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
89#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
90 | HMVMX_UPDATED_GUEST_RSP \
91 | HMVMX_UPDATED_GUEST_RFLAGS \
92 | HMVMX_UPDATED_GUEST_CR0 \
93 | HMVMX_UPDATED_GUEST_CR3 \
94 | HMVMX_UPDATED_GUEST_CR4 \
95 | HMVMX_UPDATED_GUEST_GDTR \
96 | HMVMX_UPDATED_GUEST_IDTR \
97 | HMVMX_UPDATED_GUEST_LDTR \
98 | HMVMX_UPDATED_GUEST_TR \
99 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
100 | HMVMX_UPDATED_GUEST_DEBUG \
101 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
102 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
104 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
105 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
106 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
107 | HMVMX_UPDATED_GUEST_INTR_STATE \
108 | HMVMX_UPDATED_GUEST_APIC_STATE)
109/** @} */
110
111/** @name
112 * Flags to skip redundant reads of some common VMCS fields that are not part of
113 * the guest-CPU state but are in the transient structure.
114 */
115#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
117#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
122/** @} */
123
124/** @name
125 * States of the VMCS.
126 *
127 * This does not reflect all possible VMCS states but currently only those
128 * needed for maintaining the VMCS consistently even when thread-context hooks
129 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
130 */
131#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
132#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
133#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
134/** @} */
135
136/**
137 * Exception bitmap mask for real-mode guests (real-on-v86).
138 *
139 * We need to intercept all exceptions manually except:
140 * - #NM, #MF handled in hmR0VmxLoadSharedCR0().
141 * - #DB handled in hmR0VmxLoadSharedDebugState().
142 * - #PF need not be intercepted even in real-mode if we have Nested Paging
143 * support.
144 */
145#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
146 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
147 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
148 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
149 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
150 /* RT_BIT(X86_XCPT_MF) */ | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
151 | RT_BIT(X86_XCPT_XF))
152
153/**
154 * Exception bitmap mask for all contributory exceptions.
155 *
156 * Page fault is deliberately excluded here as it's conditional as to whether
157 * it's contributory or benign. Page faults are handled separately.
158 */
159#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
160 | RT_BIT(X86_XCPT_DE))
161
162/** Maximum VM-instruction error number. */
163#define HMVMX_INSTR_ERROR_MAX 28
164
165/** Profiling macro. */
166#ifdef HM_PROFILE_EXIT_DISPATCH
167# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
168# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
169#else
170# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
171# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
172#endif
173
174/** Assert that preemption is disabled or covered by thread-context hooks. */
175#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
176 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
177
178/** Assert that we haven't migrated CPUs when thread-context hooks are not
179 * used. */
180#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
181 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
182 ("Illegal migration! Entered on CPU %u Current %u\n", \
183 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
184
185/** Helper macro for VM-exit handlers called unexpectedly. */
186#define HMVMX_RETURN_UNEXPECTED_EXIT() \
187 do { \
188 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
189 return VERR_VMX_UNEXPECTED_EXIT; \
190 } while (0)
191
192
193/*********************************************************************************************************************************
194* Structures and Typedefs *
195*********************************************************************************************************************************/
196/**
197 * VMX transient state.
198 *
199 * A state structure for holding miscellaneous information across
200 * VMX non-root operation and restored after the transition.
201 */
202typedef struct VMXTRANSIENT
203{
204 /** The host's rflags/eflags. */
205 RTCCUINTREG fEFlags;
206#if HC_ARCH_BITS == 32
207 uint32_t u32Alignment0;
208#endif
209 /** The guest's TPR value used for TPR shadowing. */
210 uint8_t u8GuestTpr;
211 /** Alignment. */
212 uint8_t abAlignment0[7];
213
214 /** The basic VM-exit reason. */
215 uint16_t uExitReason;
216 /** Alignment. */
217 uint16_t u16Alignment0;
218 /** The VM-exit interruption error code. */
219 uint32_t uExitIntErrorCode;
220 /** The VM-exit exit code qualification. */
221 uint64_t uExitQualification;
222
223 /** The VM-exit interruption-information field. */
224 uint32_t uExitIntInfo;
225 /** The VM-exit instruction-length field. */
226 uint32_t cbInstr;
227 /** The VM-exit instruction-information field. */
228 union
229 {
230 /** Plain unsigned int representation. */
231 uint32_t u;
232 /** INS and OUTS information. */
233 struct
234 {
235 uint32_t u6Reserved0 : 7;
236 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
237 uint32_t u3AddrSize : 3;
238 uint32_t u5Reserved1 : 5;
239 /** The segment register (X86_SREG_XXX). */
240 uint32_t iSegReg : 3;
241 uint32_t uReserved2 : 14;
242 } StrIo;
243 } ExitInstrInfo;
244 /** Whether the VM-entry failed or not. */
245 bool fVMEntryFailed;
246 /** Alignment. */
247 uint8_t abAlignment1[3];
248
249 /** The VM-entry interruption-information field. */
250 uint32_t uEntryIntInfo;
251 /** The VM-entry exception error code field. */
252 uint32_t uEntryXcptErrorCode;
253 /** The VM-entry instruction length field. */
254 uint32_t cbEntryInstr;
255
256 /** IDT-vectoring information field. */
257 uint32_t uIdtVectoringInfo;
258 /** IDT-vectoring error code. */
259 uint32_t uIdtVectoringErrorCode;
260
261 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
262 uint32_t fVmcsFieldsRead;
263
264 /** Whether the guest FPU was active at the time of VM-exit. */
265 bool fWasGuestFPUStateActive;
266 /** Whether the guest debug state was active at the time of VM-exit. */
267 bool fWasGuestDebugStateActive;
268 /** Whether the hyper debug state was active at the time of VM-exit. */
269 bool fWasHyperDebugStateActive;
270 /** Whether TSC-offsetting should be setup before VM-entry. */
271 bool fUpdateTscOffsettingAndPreemptTimer;
272 /** Whether the VM-exit was caused by a page-fault during delivery of a
273 * contributory exception or a page-fault. */
274 bool fVectoringDoublePF;
275 /** Whether the VM-exit was caused by a page-fault during delivery of an
276 * external interrupt or NMI. */
277 bool fVectoringPF;
278} VMXTRANSIENT;
279AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284/** Pointer to VMX transient state. */
285typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288/**
289 * MSR-bitmap read permissions.
290 */
291typedef enum VMXMSREXITREAD
292{
293 /** Reading this MSR causes a VM-exit. */
294 VMXMSREXIT_INTERCEPT_READ = 0xb,
295 /** Reading this MSR does not cause a VM-exit. */
296 VMXMSREXIT_PASSTHRU_READ
297} VMXMSREXITREAD;
298/** Pointer to MSR-bitmap read permissions. */
299typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301/**
302 * MSR-bitmap write permissions.
303 */
304typedef enum VMXMSREXITWRITE
305{
306 /** Writing to this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308 /** Writing to this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_WRITE
310} VMXMSREXITWRITE;
311/** Pointer to MSR-bitmap write permissions. */
312typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315/**
316 * VMX VM-exit handler.
317 *
318 * @returns VBox status code.
319 * @param pVCpu Pointer to the VMCPU.
320 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
321 * out-of-sync. Make sure to update the required
322 * fields before using them.
323 * @param pVmxTransient Pointer to the VMX-transient structure.
324 */
325#ifndef HMVMX_USE_FUNCTION_TABLE
326typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327#else
328typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329/** Pointer to VM-exit handler. */
330typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331#endif
332
333
334/*********************************************************************************************************************************
335* Internal Functions *
336*********************************************************************************************************************************/
337static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
338static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
339static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
340static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
341 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
342 bool fStepping, uint32_t *puIntState);
343#if HC_ARCH_BITS == 32
344static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
345#endif
346#ifndef HMVMX_USE_FUNCTION_TABLE
347DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
348# define HMVMX_EXIT_DECL static int
349#else
350# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
351#endif
352DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
353 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
354
355/** @name VM-exit handlers.
356 * @{
357 */
358static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
359static FNVMXEXITHANDLER hmR0VmxExitExtInt;
360static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
361static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
362static FNVMXEXITHANDLER hmR0VmxExitSipi;
363static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
364static FNVMXEXITHANDLER hmR0VmxExitSmi;
365static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
366static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
367static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
368static FNVMXEXITHANDLER hmR0VmxExitCpuid;
369static FNVMXEXITHANDLER hmR0VmxExitGetsec;
370static FNVMXEXITHANDLER hmR0VmxExitHlt;
371static FNVMXEXITHANDLER hmR0VmxExitInvd;
372static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
373static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
374static FNVMXEXITHANDLER hmR0VmxExitVmcall;
375static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
376static FNVMXEXITHANDLER hmR0VmxExitRsm;
377static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
378static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
379static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
380static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
381static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
382static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
383static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
384static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
385static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
386static FNVMXEXITHANDLER hmR0VmxExitMwait;
387static FNVMXEXITHANDLER hmR0VmxExitMtf;
388static FNVMXEXITHANDLER hmR0VmxExitMonitor;
389static FNVMXEXITHANDLER hmR0VmxExitPause;
390static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
391static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
392static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
393static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
394static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
395static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
396static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
397static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
398static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
399static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
400static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
401static FNVMXEXITHANDLER hmR0VmxExitRdrand;
402static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
403/** @} */
404
405static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
412static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413#endif
414static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
415
416
417/*********************************************************************************************************************************
418* Global Variables *
419*********************************************************************************************************************************/
420#ifdef HMVMX_USE_FUNCTION_TABLE
421
422/**
423 * VMX_EXIT dispatch table.
424 */
425static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
426{
427 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
428 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
429 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
430 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
431 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
432 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
433 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
434 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
435 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
436 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
437 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
438 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
439 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
440 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
441 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
442 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
443 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
444 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
445 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
446 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
447 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
448 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
449 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
450 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
451 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
452 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
453 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
454 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
455 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
456 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
457 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
458 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
459 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
460 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
461 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
462 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
463 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
464 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
465 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
466 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
467 /* 40 UNDEFINED */ hmR0VmxExitPause,
468 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
469 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
470 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
471 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
472 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
473 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
474 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
475 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
476 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
477 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
478 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
479 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
480 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
481 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
482 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
483 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
484 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
485 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
486 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
487 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
488 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
489 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
490 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
491 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
492};
493#endif /* HMVMX_USE_FUNCTION_TABLE */
494
495#ifdef VBOX_STRICT
496static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
497{
498 /* 0 */ "(Not Used)",
499 /* 1 */ "VMCALL executed in VMX root operation.",
500 /* 2 */ "VMCLEAR with invalid physical address.",
501 /* 3 */ "VMCLEAR with VMXON pointer.",
502 /* 4 */ "VMLAUNCH with non-clear VMCS.",
503 /* 5 */ "VMRESUME with non-launched VMCS.",
504 /* 6 */ "VMRESUME after VMXOFF",
505 /* 7 */ "VM-entry with invalid control fields.",
506 /* 8 */ "VM-entry with invalid host state fields.",
507 /* 9 */ "VMPTRLD with invalid physical address.",
508 /* 10 */ "VMPTRLD with VMXON pointer.",
509 /* 11 */ "VMPTRLD with incorrect revision identifier.",
510 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
511 /* 13 */ "VMWRITE to read-only VMCS component.",
512 /* 14 */ "(Not Used)",
513 /* 15 */ "VMXON executed in VMX root operation.",
514 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
515 /* 17 */ "VM-entry with non-launched executing VMCS.",
516 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
517 /* 19 */ "VMCALL with non-clear VMCS.",
518 /* 20 */ "VMCALL with invalid VM-exit control fields.",
519 /* 21 */ "(Not Used)",
520 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
521 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
522 /* 24 */ "VMCALL with invalid SMM-monitor features.",
523 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
524 /* 26 */ "VM-entry with events blocked by MOV SS.",
525 /* 27 */ "(Not Used)",
526 /* 28 */ "Invalid operand to INVEPT/INVVPID."
527};
528#endif /* VBOX_STRICT */
529
530
531
532/**
533 * Updates the VM's last error record. If there was a VMX instruction error,
534 * reads the error data from the VMCS and updates VCPU's last error record as
535 * well.
536 *
537 * @param pVM Pointer to the VM.
538 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
539 * VERR_VMX_UNABLE_TO_START_VM or
540 * VERR_VMX_INVALID_VMCS_FIELD).
541 * @param rc The error code.
542 */
543static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
544{
545 AssertPtr(pVM);
546 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
547 || rc == VERR_VMX_UNABLE_TO_START_VM)
548 {
549 AssertPtrReturnVoid(pVCpu);
550 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
551 }
552 pVM->hm.s.lLastError = rc;
553}
554
555
556/**
557 * Reads the VM-entry interruption-information field from the VMCS into the VMX
558 * transient structure.
559 *
560 * @returns VBox status code.
561 * @param pVmxTransient Pointer to the VMX transient structure.
562 *
563 * @remarks No-long-jump zone!!!
564 */
565DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
566{
567 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
568 AssertRCReturn(rc, rc);
569 return VINF_SUCCESS;
570}
571
572
573/**
574 * Reads the VM-entry exception error code field from the VMCS into
575 * the VMX transient structure.
576 *
577 * @returns VBox status code.
578 * @param pVmxTransient Pointer to the VMX transient structure.
579 *
580 * @remarks No-long-jump zone!!!
581 */
582DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
583{
584 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
585 AssertRCReturn(rc, rc);
586 return VINF_SUCCESS;
587}
588
589
590/**
591 * Reads the VM-entry exception error code field from the VMCS into
592 * the VMX transient structure.
593 *
594 * @returns VBox status code.
595 * @param pVmxTransient Pointer to the VMX transient structure.
596 *
597 * @remarks No-long-jump zone!!!
598 */
599DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
600{
601 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
602 AssertRCReturn(rc, rc);
603 return VINF_SUCCESS;
604}
605
606
607/**
608 * Reads the VM-exit interruption-information field from the VMCS into the VMX
609 * transient structure.
610 *
611 * @returns VBox status code.
612 * @param pVmxTransient Pointer to the VMX transient structure.
613 */
614DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
615{
616 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
617 {
618 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
619 AssertRCReturn(rc, rc);
620 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
621 }
622 return VINF_SUCCESS;
623}
624
625
626/**
627 * Reads the VM-exit interruption error code from the VMCS into the VMX
628 * transient structure.
629 *
630 * @returns VBox status code.
631 * @param pVmxTransient Pointer to the VMX transient structure.
632 */
633DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
634{
635 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
636 {
637 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
638 AssertRCReturn(rc, rc);
639 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
640 }
641 return VINF_SUCCESS;
642}
643
644
645/**
646 * Reads the VM-exit instruction length field from the VMCS into the VMX
647 * transient structure.
648 *
649 * @returns VBox status code.
650 * @param pVCpu Pointer to the VMCPU.
651 * @param pVmxTransient Pointer to the VMX transient structure.
652 */
653DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
654{
655 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
656 {
657 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
658 AssertRCReturn(rc, rc);
659 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
660 }
661 return VINF_SUCCESS;
662}
663
664
665/**
666 * Reads the VM-exit instruction-information field from the VMCS into
667 * the VMX transient structure.
668 *
669 * @returns VBox status code.
670 * @param pVmxTransient Pointer to the VMX transient structure.
671 */
672DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
673{
674 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
675 {
676 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
677 AssertRCReturn(rc, rc);
678 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
679 }
680 return VINF_SUCCESS;
681}
682
683
684/**
685 * Reads the exit code qualification from the VMCS into the VMX transient
686 * structure.
687 *
688 * @returns VBox status code.
689 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
690 * case).
691 * @param pVmxTransient Pointer to the VMX transient structure.
692 */
693DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
694{
695 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
696 {
697 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
698 AssertRCReturn(rc, rc);
699 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
700 }
701 return VINF_SUCCESS;
702}
703
704
705/**
706 * Reads the IDT-vectoring information field from the VMCS into the VMX
707 * transient structure.
708 *
709 * @returns VBox status code.
710 * @param pVmxTransient Pointer to the VMX transient structure.
711 *
712 * @remarks No-long-jump zone!!!
713 */
714DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
715{
716 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
717 {
718 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
719 AssertRCReturn(rc, rc);
720 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
721 }
722 return VINF_SUCCESS;
723}
724
725
726/**
727 * Reads the IDT-vectoring error code from the VMCS into the VMX
728 * transient structure.
729 *
730 * @returns VBox status code.
731 * @param pVmxTransient Pointer to the VMX transient structure.
732 */
733DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
734{
735 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
736 {
737 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
738 AssertRCReturn(rc, rc);
739 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
740 }
741 return VINF_SUCCESS;
742}
743
744
745/**
746 * Enters VMX root mode operation on the current CPU.
747 *
748 * @returns VBox status code.
749 * @param pVM Pointer to the VM (optional, can be NULL, after
750 * a resume).
751 * @param HCPhysCpuPage Physical address of the VMXON region.
752 * @param pvCpuPage Pointer to the VMXON region.
753 */
754static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
755{
756 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
757 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
758 Assert(pvCpuPage);
759 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
760
761 if (pVM)
762 {
763 /* Write the VMCS revision dword to the VMXON region. */
764 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
765 }
766
767 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
768 RTCCUINTREG fEFlags = ASMIntDisableFlags();
769
770 /* Enable the VMX bit in CR4 if necessary. */
771 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
772
773 /* Enter VMX root mode. */
774 int rc = VMXEnable(HCPhysCpuPage);
775 if (RT_FAILURE(rc))
776 {
777 if (!(uOldCr4 & X86_CR4_VMXE))
778 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
779
780 if (pVM)
781 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
782 }
783
784 /* Restore interrupts. */
785 ASMSetFlags(fEFlags);
786 return rc;
787}
788
789
790/**
791 * Exits VMX root mode operation on the current CPU.
792 *
793 * @returns VBox status code.
794 */
795static int hmR0VmxLeaveRootMode(void)
796{
797 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
798
799 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
800 RTCCUINTREG fEFlags = ASMIntDisableFlags();
801
802 /* If we're for some reason not in VMX root mode, then don't leave it. */
803 RTCCUINTREG uHostCR4 = ASMGetCR4();
804
805 int rc;
806 if (uHostCR4 & X86_CR4_VMXE)
807 {
808 /* Exit VMX root mode and clear the VMX bit in CR4. */
809 VMXDisable();
810 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
811 rc = VINF_SUCCESS;
812 }
813 else
814 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
815
816 /* Restore interrupts. */
817 ASMSetFlags(fEFlags);
818 return rc;
819}
820
821
822/**
823 * Allocates and maps one physically contiguous page. The allocated page is
824 * zero'd out. (Used by various VT-x structures).
825 *
826 * @returns IPRT status code.
827 * @param pMemObj Pointer to the ring-0 memory object.
828 * @param ppVirt Where to store the virtual address of the
829 * allocation.
830 * @param pPhys Where to store the physical address of the
831 * allocation.
832 */
833DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
834{
835 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
836 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
837 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
838
839 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
840 if (RT_FAILURE(rc))
841 return rc;
842 *ppVirt = RTR0MemObjAddress(*pMemObj);
843 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
844 ASMMemZero32(*ppVirt, PAGE_SIZE);
845 return VINF_SUCCESS;
846}
847
848
849/**
850 * Frees and unmaps an allocated physical page.
851 *
852 * @param pMemObj Pointer to the ring-0 memory object.
853 * @param ppVirt Where to re-initialize the virtual address of
854 * allocation as 0.
855 * @param pHCPhys Where to re-initialize the physical address of the
856 * allocation as 0.
857 */
858DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
859{
860 AssertPtr(pMemObj);
861 AssertPtr(ppVirt);
862 AssertPtr(pHCPhys);
863 if (*pMemObj != NIL_RTR0MEMOBJ)
864 {
865 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
866 AssertRC(rc);
867 *pMemObj = NIL_RTR0MEMOBJ;
868 *ppVirt = 0;
869 *pHCPhys = 0;
870 }
871}
872
873
874/**
875 * Worker function to free VT-x related structures.
876 *
877 * @returns IPRT status code.
878 * @param pVM Pointer to the VM.
879 */
880static void hmR0VmxStructsFree(PVM pVM)
881{
882 for (VMCPUID i = 0; i < pVM->cCpus; i++)
883 {
884 PVMCPU pVCpu = &pVM->aCpus[i];
885 AssertPtr(pVCpu);
886
887 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
888 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
889
890 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
891 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
892
893 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
894 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
895 }
896
897 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
898#ifdef VBOX_WITH_CRASHDUMP_MAGIC
899 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
900#endif
901}
902
903
904/**
905 * Worker function to allocate VT-x related VM structures.
906 *
907 * @returns IPRT status code.
908 * @param pVM Pointer to the VM.
909 */
910static int hmR0VmxStructsAlloc(PVM pVM)
911{
912 /*
913 * Initialize members up-front so we can cleanup properly on allocation failure.
914 */
915#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
916 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
917 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
918 pVM->hm.s.vmx.HCPhys##a_Name = 0;
919
920#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
921 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
922 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
923 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
924
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
927#endif
928 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
929
930 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
931 for (VMCPUID i = 0; i < pVM->cCpus; i++)
932 {
933 PVMCPU pVCpu = &pVM->aCpus[i];
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
935 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
936 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
937 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
938 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
939 }
940#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
941#undef VMXLOCAL_INIT_VM_MEMOBJ
942
943 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
944 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
945 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
946 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
947
948 /*
949 * Allocate all the VT-x structures.
950 */
951 int rc = VINF_SUCCESS;
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
957 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
958#endif
959
960 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
961 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
962 {
963 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
964 &pVM->hm.s.vmx.HCPhysApicAccess);
965 if (RT_FAILURE(rc))
966 goto cleanup;
967 }
968
969 /*
970 * Initialize per-VCPU VT-x structures.
971 */
972 for (VMCPUID i = 0; i < pVM->cCpus; i++)
973 {
974 PVMCPU pVCpu = &pVM->aCpus[i];
975 AssertPtr(pVCpu);
976
977 /* Allocate the VM control structure (VMCS). */
978 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
979 if (RT_FAILURE(rc))
980 goto cleanup;
981
982 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
983 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
984 {
985 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
986 &pVCpu->hm.s.vmx.HCPhysVirtApic);
987 if (RT_FAILURE(rc))
988 goto cleanup;
989 }
990
991 /*
992 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
993 * transparent accesses of specific MSRs.
994 *
995 * If the condition for enabling MSR bitmaps changes here, don't forget to
996 * update HMAreMsrBitmapsAvailable().
997 */
998 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
999 {
1000 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1001 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1002 if (RT_FAILURE(rc))
1003 goto cleanup;
1004 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1005 }
1006
1007 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1008 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1009 if (RT_FAILURE(rc))
1010 goto cleanup;
1011
1012 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1013 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016 }
1017
1018 return VINF_SUCCESS;
1019
1020cleanup:
1021 hmR0VmxStructsFree(pVM);
1022 return rc;
1023}
1024
1025
1026/**
1027 * Does global VT-x initialization (called during module initialization).
1028 *
1029 * @returns VBox status code.
1030 */
1031VMMR0DECL(int) VMXR0GlobalInit(void)
1032{
1033#ifdef HMVMX_USE_FUNCTION_TABLE
1034 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1035# ifdef VBOX_STRICT
1036 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1037 Assert(g_apfnVMExitHandlers[i]);
1038# endif
1039#endif
1040 return VINF_SUCCESS;
1041}
1042
1043
1044/**
1045 * Does global VT-x termination (called during module termination).
1046 */
1047VMMR0DECL(void) VMXR0GlobalTerm()
1048{
1049 /* Nothing to do currently. */
1050}
1051
1052
1053/**
1054 * Sets up and activates VT-x on the current CPU.
1055 *
1056 * @returns VBox status code.
1057 * @param pCpu Pointer to the global CPU info struct.
1058 * @param pVM Pointer to the VM (can be NULL after a host resume
1059 * operation).
1060 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1061 * fEnabledByHost is true).
1062 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1063 * @a fEnabledByHost is true).
1064 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1065 * enable VT-x on the host.
1066 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1067 */
1068VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1069 void *pvMsrs)
1070{
1071 Assert(pCpu);
1072 Assert(pvMsrs);
1073 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1074
1075 /* Enable VT-x if it's not already enabled by the host. */
1076 if (!fEnabledByHost)
1077 {
1078 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1079 if (RT_FAILURE(rc))
1080 return rc;
1081 }
1082
1083 /*
1084 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1085 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1086 */
1087 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1088 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1089 {
1090 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1091 pCpu->fFlushAsidBeforeUse = false;
1092 }
1093 else
1094 pCpu->fFlushAsidBeforeUse = true;
1095
1096 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1097 ++pCpu->cTlbFlushes;
1098
1099 return VINF_SUCCESS;
1100}
1101
1102
1103/**
1104 * Deactivates VT-x on the current CPU.
1105 *
1106 * @returns VBox status code.
1107 * @param pCpu Pointer to the global CPU info struct.
1108 * @param pvCpuPage Pointer to the VMXON region.
1109 * @param HCPhysCpuPage Physical address of the VMXON region.
1110 *
1111 * @remarks This function should never be called when SUPR0EnableVTx() or
1112 * similar was used to enable VT-x on the host.
1113 */
1114VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1115{
1116 NOREF(pCpu);
1117 NOREF(pvCpuPage);
1118 NOREF(HCPhysCpuPage);
1119
1120 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1121 return hmR0VmxLeaveRootMode();
1122}
1123
1124
1125/**
1126 * Sets the permission bits for the specified MSR in the MSR bitmap.
1127 *
1128 * @param pVCpu Pointer to the VMCPU.
1129 * @param uMSR The MSR value.
1130 * @param enmRead Whether reading this MSR causes a VM-exit.
1131 * @param enmWrite Whether writing this MSR causes a VM-exit.
1132 */
1133static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1134{
1135 int32_t iBit;
1136 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1137
1138 /*
1139 * Layout:
1140 * 0x000 - 0x3ff - Low MSR read bits
1141 * 0x400 - 0x7ff - High MSR read bits
1142 * 0x800 - 0xbff - Low MSR write bits
1143 * 0xc00 - 0xfff - High MSR write bits
1144 */
1145 if (uMsr <= 0x00001FFF)
1146 iBit = uMsr;
1147 else if ( uMsr >= 0xC0000000
1148 && uMsr <= 0xC0001FFF)
1149 {
1150 iBit = (uMsr - 0xC0000000);
1151 pbMsrBitmap += 0x400;
1152 }
1153 else
1154 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1155
1156 Assert(iBit <= 0x1fff);
1157 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1158 ASMBitSet(pbMsrBitmap, iBit);
1159 else
1160 ASMBitClear(pbMsrBitmap, iBit);
1161
1162 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1163 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1164 else
1165 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1166}
1167
1168
1169#ifdef VBOX_STRICT
1170/**
1171 * Gets the permission bits for the specified MSR in the MSR bitmap.
1172 *
1173 * @returns VBox status code.
1174 * @retval VINF_SUCCESS if the specified MSR is found.
1175 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1176 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1177 *
1178 * @param pVCpu Pointer to the VMCPU.
1179 * @param uMsr The MSR.
1180 * @param penmRead Where to store the read permissions.
1181 * @param penmWrite Where to store the write permissions.
1182 */
1183static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1184{
1185 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1186 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1187 int32_t iBit;
1188 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1189
1190 /* See hmR0VmxSetMsrPermission() for the layout. */
1191 if (uMsr <= 0x00001FFF)
1192 iBit = uMsr;
1193 else if ( uMsr >= 0xC0000000
1194 && uMsr <= 0xC0001FFF)
1195 {
1196 iBit = (uMsr - 0xC0000000);
1197 pbMsrBitmap += 0x400;
1198 }
1199 else
1200 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1201
1202 Assert(iBit <= 0x1fff);
1203 if (ASMBitTest(pbMsrBitmap, iBit))
1204 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1205 else
1206 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1207
1208 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1209 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1210 else
1211 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1212 return VINF_SUCCESS;
1213}
1214#endif /* VBOX_STRICT */
1215
1216
1217/**
1218 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1219 * area.
1220 *
1221 * @returns VBox status code.
1222 * @param pVCpu Pointer to the VMCPU.
1223 * @param cMsrs The number of MSRs.
1224 */
1225DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1226{
1227 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1228 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1229 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1230 {
1231 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1232 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1233 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1234 }
1235
1236 /* Update number of guest MSRs to load/store across the world-switch. */
1237 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1238 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1239
1240 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1241 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1242
1243 /* Update the VCPU's copy of the MSR count. */
1244 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1245
1246 return VINF_SUCCESS;
1247}
1248
1249
1250/**
1251 * Adds a new (or updates the value of an existing) guest/host MSR
1252 * pair to be swapped during the world-switch as part of the
1253 * auto-load/store MSR area in the VMCS.
1254 *
1255 * @returns VBox status code.
1256 * @param pVCpu Pointer to the VMCPU.
1257 * @param uMsr The MSR.
1258 * @param uGuestMsr Value of the guest MSR.
1259 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1260 * necessary.
1261 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1262 * its value was updated. Optional, can be NULL.
1263 */
1264static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1265 bool *pfAddedAndUpdated)
1266{
1267 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1268 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1269 uint32_t i;
1270 for (i = 0; i < cMsrs; i++)
1271 {
1272 if (pGuestMsr->u32Msr == uMsr)
1273 break;
1274 pGuestMsr++;
1275 }
1276
1277 bool fAdded = false;
1278 if (i == cMsrs)
1279 {
1280 ++cMsrs;
1281 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1282 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1283
1284 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1285 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1286 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1287
1288 fAdded = true;
1289 }
1290
1291 /* Update the MSR values in the auto-load/store MSR area. */
1292 pGuestMsr->u32Msr = uMsr;
1293 pGuestMsr->u64Value = uGuestMsrValue;
1294
1295 /* Create/update the MSR slot in the host MSR area. */
1296 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1297 pHostMsr += i;
1298 pHostMsr->u32Msr = uMsr;
1299
1300 /*
1301 * Update the host MSR only when requested by the caller AND when we're
1302 * adding it to the auto-load/store area. Otherwise, it would have been
1303 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1304 */
1305 bool fUpdatedMsrValue = false;
1306 if ( fAdded
1307 && fUpdateHostMsr)
1308 {
1309 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1310 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1311 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1312 fUpdatedMsrValue = true;
1313 }
1314
1315 if (pfAddedAndUpdated)
1316 *pfAddedAndUpdated = fUpdatedMsrValue;
1317 return VINF_SUCCESS;
1318}
1319
1320
1321/**
1322 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1323 * auto-load/store MSR area in the VMCS.
1324 *
1325 * @returns VBox status code.
1326 * @param pVCpu Pointer to the VMCPU.
1327 * @param uMsr The MSR.
1328 */
1329static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1330{
1331 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1332 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1333 for (uint32_t i = 0; i < cMsrs; i++)
1334 {
1335 /* Find the MSR. */
1336 if (pGuestMsr->u32Msr == uMsr)
1337 {
1338 /* If it's the last MSR, simply reduce the count. */
1339 if (i == cMsrs - 1)
1340 {
1341 --cMsrs;
1342 break;
1343 }
1344
1345 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1346 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1347 pLastGuestMsr += cMsrs - 1;
1348 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1349 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1350
1351 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1352 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1353 pLastHostMsr += cMsrs - 1;
1354 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1355 pHostMsr->u64Value = pLastHostMsr->u64Value;
1356 --cMsrs;
1357 break;
1358 }
1359 pGuestMsr++;
1360 }
1361
1362 /* Update the VMCS if the count changed (meaning the MSR was found). */
1363 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1364 {
1365 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1366 AssertRCReturn(rc, rc);
1367
1368 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1369 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1370 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1371
1372 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1373 return VINF_SUCCESS;
1374 }
1375
1376 return VERR_NOT_FOUND;
1377}
1378
1379
1380/**
1381 * Checks if the specified guest MSR is part of the auto-load/store area in
1382 * the VMCS.
1383 *
1384 * @returns true if found, false otherwise.
1385 * @param pVCpu Pointer to the VMCPU.
1386 * @param uMsr The MSR to find.
1387 */
1388static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1389{
1390 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1391 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1392
1393 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1394 {
1395 if (pGuestMsr->u32Msr == uMsr)
1396 return true;
1397 }
1398 return false;
1399}
1400
1401
1402/**
1403 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1404 *
1405 * @param pVCpu Pointer to the VMCPU.
1406 *
1407 * @remarks No-long-jump zone!!!
1408 */
1409static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1410{
1411 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1412 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1413 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1414 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1415
1416 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1417 {
1418 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1419
1420 /*
1421 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1422 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1423 */
1424 if (pHostMsr->u32Msr == MSR_K6_EFER)
1425 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1426 else
1427 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1428 }
1429
1430 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1431}
1432
1433
1434#if HC_ARCH_BITS == 64
1435/**
1436 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1437 * perform lazy restoration of the host MSRs while leaving VT-x.
1438 *
1439 * @param pVCpu Pointer to the VMCPU.
1440 *
1441 * @remarks No-long-jump zone!!!
1442 */
1443static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1444{
1445 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1446
1447 /*
1448 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1449 */
1450 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1451 {
1452 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1453 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1454 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1455 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1456 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1457 }
1458}
1459
1460
1461/**
1462 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1463 * lazily while leaving VT-x.
1464 *
1465 * @returns true if it does, false otherwise.
1466 * @param pVCpu Pointer to the VMCPU.
1467 * @param uMsr The MSR to check.
1468 */
1469static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1470{
1471 NOREF(pVCpu);
1472 switch (uMsr)
1473 {
1474 case MSR_K8_LSTAR:
1475 case MSR_K6_STAR:
1476 case MSR_K8_SF_MASK:
1477 case MSR_K8_KERNEL_GS_BASE:
1478 return true;
1479 }
1480 return false;
1481}
1482
1483
1484/**
1485 * Saves a set of guest MSRs back into the guest-CPU context.
1486 *
1487 * @param pVCpu Pointer to the VMCPU.
1488 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1489 * out-of-sync. Make sure to update the required fields
1490 * before using them.
1491 *
1492 * @remarks No-long-jump zone!!!
1493 */
1494static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1495{
1496 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1497 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1498
1499 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1500 {
1501 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1502 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1503 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1504 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1505 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1506 }
1507}
1508
1509
1510/**
1511 * Loads a set of guests MSRs to allow read/passthru to the guest.
1512 *
1513 * The name of this function is slightly confusing. This function does NOT
1514 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1515 * common prefix for functions dealing with "lazy restoration" of the shared
1516 * MSRs.
1517 *
1518 * @param pVCpu Pointer to the VMCPU.
1519 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1520 * out-of-sync. Make sure to update the required fields
1521 * before using them.
1522 *
1523 * @remarks No-long-jump zone!!!
1524 */
1525static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1526{
1527 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1528 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1529
1530#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1531 do { \
1532 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1533 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1534 else \
1535 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1536 } while (0)
1537
1538 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1539 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1540 {
1541 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1542 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1543 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1544 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1545 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1546 }
1547 else
1548 {
1549 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1550 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1551 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1552 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1553 }
1554
1555#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1556}
1557
1558
1559/**
1560 * Performs lazy restoration of the set of host MSRs if they were previously
1561 * loaded with guest MSR values.
1562 *
1563 * @param pVCpu Pointer to the VMCPU.
1564 *
1565 * @remarks No-long-jump zone!!!
1566 * @remarks The guest MSRs should have been saved back into the guest-CPU
1567 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1568 */
1569static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1570{
1571 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1572 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1573
1574 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1575 {
1576 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1577 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1578 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1579 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1580 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1581 }
1582 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1583}
1584#endif /* HC_ARCH_BITS == 64 */
1585
1586
1587/**
1588 * Verifies that our cached values of the VMCS controls are all
1589 * consistent with what's actually present in the VMCS.
1590 *
1591 * @returns VBox status code.
1592 * @param pVCpu Pointer to the VMCPU.
1593 */
1594static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1595{
1596 uint32_t u32Val;
1597 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1598 AssertRCReturn(rc, rc);
1599 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1600 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1601
1602 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1603 AssertRCReturn(rc, rc);
1604 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1605 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1606
1607 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1608 AssertRCReturn(rc, rc);
1609 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1610 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1611
1612 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1613 AssertRCReturn(rc, rc);
1614 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1615 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1616
1617 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1618 {
1619 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1620 AssertRCReturn(rc, rc);
1621 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1622 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1623 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1624 }
1625
1626 return VINF_SUCCESS;
1627}
1628
1629
1630#ifdef VBOX_STRICT
1631/**
1632 * Verifies that our cached host EFER value has not changed
1633 * since we cached it.
1634 *
1635 * @param pVCpu Pointer to the VMCPU.
1636 */
1637static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1638{
1639 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1640
1641 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1642 {
1643 uint64_t u64Val;
1644 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1645 AssertRC(rc);
1646
1647 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1648 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1649 }
1650}
1651
1652
1653/**
1654 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1655 * VMCS are correct.
1656 *
1657 * @param pVCpu Pointer to the VMCPU.
1658 */
1659static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1660{
1661 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1662
1663 /* Verify MSR counts in the VMCS are what we think it should be. */
1664 uint32_t cMsrs;
1665 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1666 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1667
1668 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1669 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1670
1671 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1672 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1673
1674 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1675 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1676 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1677 {
1678 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1679 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1680 pGuestMsr->u32Msr, cMsrs));
1681
1682 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1683 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1684 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1685
1686 /* Verify that the permissions are as expected in the MSR bitmap. */
1687 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1688 {
1689 VMXMSREXITREAD enmRead;
1690 VMXMSREXITWRITE enmWrite;
1691 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1692 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1693 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1694 {
1695 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1696 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1697 }
1698 else
1699 {
1700 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1701 pGuestMsr->u32Msr, cMsrs));
1702 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1703 pGuestMsr->u32Msr, cMsrs));
1704 }
1705 }
1706 }
1707}
1708#endif /* VBOX_STRICT */
1709
1710
1711/**
1712 * Flushes the TLB using EPT.
1713 *
1714 * @returns VBox status code.
1715 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1716 * enmFlush).
1717 * @param enmFlush Type of flush.
1718 *
1719 * @remarks Caller is responsible for making sure this function is called only
1720 * when NestedPaging is supported and providing @a enmFlush that is
1721 * supported by the CPU.
1722 * @remarks Can be called with interrupts disabled.
1723 */
1724static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1725{
1726 uint64_t au64Descriptor[2];
1727 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1728 au64Descriptor[0] = 0;
1729 else
1730 {
1731 Assert(pVCpu);
1732 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1733 }
1734 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1735
1736 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1737 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1738 rc));
1739 if ( RT_SUCCESS(rc)
1740 && pVCpu)
1741 {
1742 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1743 }
1744}
1745
1746
1747/**
1748 * Flushes the TLB using VPID.
1749 *
1750 * @returns VBox status code.
1751 * @param pVM Pointer to the VM.
1752 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1753 * enmFlush).
1754 * @param enmFlush Type of flush.
1755 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1756 * on @a enmFlush).
1757 *
1758 * @remarks Can be called with interrupts disabled.
1759 */
1760static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1761{
1762 NOREF(pVM);
1763 AssertPtr(pVM);
1764 Assert(pVM->hm.s.vmx.fVpid);
1765
1766 uint64_t au64Descriptor[2];
1767 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1768 {
1769 au64Descriptor[0] = 0;
1770 au64Descriptor[1] = 0;
1771 }
1772 else
1773 {
1774 AssertPtr(pVCpu);
1775 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1776 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1777 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1778 au64Descriptor[1] = GCPtr;
1779 }
1780
1781 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1782 AssertMsg(rc == VINF_SUCCESS,
1783 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1784 if ( RT_SUCCESS(rc)
1785 && pVCpu)
1786 {
1787 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1788 }
1789}
1790
1791
1792/**
1793 * Invalidates a guest page by guest virtual address. Only relevant for
1794 * EPT/VPID, otherwise there is nothing really to invalidate.
1795 *
1796 * @returns VBox status code.
1797 * @param pVM Pointer to the VM.
1798 * @param pVCpu Pointer to the VMCPU.
1799 * @param GCVirt Guest virtual address of the page to invalidate.
1800 */
1801VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1802{
1803 AssertPtr(pVM);
1804 AssertPtr(pVCpu);
1805 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1806
1807 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1808 if (!fFlushPending)
1809 {
1810 /*
1811 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1812 * See @bugref{6043} and @bugref{6177}.
1813 *
1814 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1815 * function maybe called in a loop with individual addresses.
1816 */
1817 if (pVM->hm.s.vmx.fVpid)
1818 {
1819 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1820 {
1821 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1822 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1823 }
1824 else
1825 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1826 }
1827 else if (pVM->hm.s.fNestedPaging)
1828 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1829 }
1830
1831 return VINF_SUCCESS;
1832}
1833
1834
1835/**
1836 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1837 * otherwise there is nothing really to invalidate.
1838 *
1839 * @returns VBox status code.
1840 * @param pVM Pointer to the VM.
1841 * @param pVCpu Pointer to the VMCPU.
1842 * @param GCPhys Guest physical address of the page to invalidate.
1843 */
1844VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1845{
1846 NOREF(pVM); NOREF(GCPhys);
1847 LogFlowFunc(("%RGp\n", GCPhys));
1848
1849 /*
1850 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1851 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1852 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1853 */
1854 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1855 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1856 return VINF_SUCCESS;
1857}
1858
1859
1860/**
1861 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1862 * case where neither EPT nor VPID is supported by the CPU.
1863 *
1864 * @param pVM Pointer to the VM.
1865 * @param pVCpu Pointer to the VMCPU.
1866 * @param pCpu Pointer to the global HM struct.
1867 *
1868 * @remarks Called with interrupts disabled.
1869 */
1870static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1871{
1872 AssertPtr(pVCpu);
1873 AssertPtr(pCpu);
1874 NOREF(pVM);
1875
1876 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1877
1878 Assert(pCpu->idCpu != NIL_RTCPUID);
1879 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1880 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1881 pVCpu->hm.s.fForceTLBFlush = false;
1882 return;
1883}
1884
1885
1886/**
1887 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1888 *
1889 * @param pVM Pointer to the VM.
1890 * @param pVCpu Pointer to the VMCPU.
1891 * @param pCpu Pointer to the global HM CPU struct.
1892 * @remarks All references to "ASID" in this function pertains to "VPID" in
1893 * Intel's nomenclature. The reason is, to avoid confusion in compare
1894 * statements since the host-CPU copies are named "ASID".
1895 *
1896 * @remarks Called with interrupts disabled.
1897 */
1898static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1899{
1900#ifdef VBOX_WITH_STATISTICS
1901 bool fTlbFlushed = false;
1902# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1903# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1904 if (!fTlbFlushed) \
1905 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1906 } while (0)
1907#else
1908# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1909# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1910#endif
1911
1912 AssertPtr(pVM);
1913 AssertPtr(pCpu);
1914 AssertPtr(pVCpu);
1915 Assert(pCpu->idCpu != NIL_RTCPUID);
1916
1917 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1918 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1919 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1920
1921 /*
1922 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1923 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1924 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1925 */
1926 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1927 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1928 {
1929 ++pCpu->uCurrentAsid;
1930 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1931 {
1932 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1933 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1934 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1935 }
1936
1937 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1938 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1939 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1940
1941 /*
1942 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1943 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1944 */
1945 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1946 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1947 HMVMX_SET_TAGGED_TLB_FLUSHED();
1948 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1949 }
1950
1951 /* Check for explicit TLB flushes. */
1952 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1953 {
1954 /*
1955 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1956 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1957 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1958 * but not guest-physical mappings.
1959 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1960 */
1961 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1962 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1963 HMVMX_SET_TAGGED_TLB_FLUSHED();
1964 }
1965
1966 pVCpu->hm.s.fForceTLBFlush = false;
1967 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1968
1969 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1970 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1971 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1972 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1973 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1974 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1975 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1976 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1977 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1978
1979 /* Update VMCS with the VPID. */
1980 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1981 AssertRC(rc);
1982
1983#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1984}
1985
1986
1987/**
1988 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1989 *
1990 * @returns VBox status code.
1991 * @param pVM Pointer to the VM.
1992 * @param pVCpu Pointer to the VMCPU.
1993 * @param pCpu Pointer to the global HM CPU struct.
1994 *
1995 * @remarks Called with interrupts disabled.
1996 */
1997static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1998{
1999 AssertPtr(pVM);
2000 AssertPtr(pVCpu);
2001 AssertPtr(pCpu);
2002 Assert(pCpu->idCpu != NIL_RTCPUID);
2003 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2004 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2005
2006 /*
2007 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2008 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2009 */
2010 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2011 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2012 {
2013 pVCpu->hm.s.fForceTLBFlush = true;
2014 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2015 }
2016
2017 /* Check for explicit TLB flushes. */
2018 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2019 {
2020 pVCpu->hm.s.fForceTLBFlush = true;
2021 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2022 }
2023
2024 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2025 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2026
2027 if (pVCpu->hm.s.fForceTLBFlush)
2028 {
2029 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2030 pVCpu->hm.s.fForceTLBFlush = false;
2031 }
2032}
2033
2034
2035/**
2036 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2037 *
2038 * @returns VBox status code.
2039 * @param pVM Pointer to the VM.
2040 * @param pVCpu Pointer to the VMCPU.
2041 * @param pCpu Pointer to the global HM CPU struct.
2042 *
2043 * @remarks Called with interrupts disabled.
2044 */
2045static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2046{
2047 AssertPtr(pVM);
2048 AssertPtr(pVCpu);
2049 AssertPtr(pCpu);
2050 Assert(pCpu->idCpu != NIL_RTCPUID);
2051 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2052 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2053
2054 /*
2055 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2056 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2057 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2058 */
2059 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2060 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2061 {
2062 pVCpu->hm.s.fForceTLBFlush = true;
2063 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2064 }
2065
2066 /* Check for explicit TLB flushes. */
2067 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2068 {
2069 /*
2070 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2071 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2072 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2073 */
2074 pVCpu->hm.s.fForceTLBFlush = true;
2075 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2076 }
2077
2078 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2079 if (pVCpu->hm.s.fForceTLBFlush)
2080 {
2081 ++pCpu->uCurrentAsid;
2082 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2083 {
2084 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2085 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2086 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2087 }
2088
2089 pVCpu->hm.s.fForceTLBFlush = false;
2090 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2091 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2092 if (pCpu->fFlushAsidBeforeUse)
2093 {
2094 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2095 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2096 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2097 {
2098 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2099 pCpu->fFlushAsidBeforeUse = false;
2100 }
2101 else
2102 {
2103 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2104 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2105 }
2106 }
2107 }
2108
2109 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2110 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2111 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2112 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2113 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2114 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2115 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2116
2117 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2118 AssertRC(rc);
2119}
2120
2121
2122/**
2123 * Flushes the guest TLB entry based on CPU capabilities.
2124 *
2125 * @param pVCpu Pointer to the VMCPU.
2126 * @param pCpu Pointer to the global HM CPU struct.
2127 */
2128DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2129{
2130#ifdef HMVMX_ALWAYS_FLUSH_TLB
2131 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2132#endif
2133 PVM pVM = pVCpu->CTX_SUFF(pVM);
2134 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2135 {
2136 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2137 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2138 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2139 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2140 default:
2141 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2142 break;
2143 }
2144
2145 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2146}
2147
2148
2149/**
2150 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2151 * TLB entries from the host TLB before VM-entry.
2152 *
2153 * @returns VBox status code.
2154 * @param pVM Pointer to the VM.
2155 */
2156static int hmR0VmxSetupTaggedTlb(PVM pVM)
2157{
2158 /*
2159 * Determine optimal flush type for Nested Paging.
2160 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2161 * guest execution (see hmR3InitFinalizeR0()).
2162 */
2163 if (pVM->hm.s.fNestedPaging)
2164 {
2165 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2166 {
2167 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2168 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2169 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2170 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2171 else
2172 {
2173 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2174 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2175 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2176 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2177 }
2178
2179 /* Make sure the write-back cacheable memory type for EPT is supported. */
2180 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2181 {
2182 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2183 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2184 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2185 }
2186
2187 /* EPT requires a page-walk length of 4. */
2188 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2189 {
2190 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2191 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2192 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2193 }
2194 }
2195 else
2196 {
2197 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2198 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2199 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2200 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2201 }
2202 }
2203
2204 /*
2205 * Determine optimal flush type for VPID.
2206 */
2207 if (pVM->hm.s.vmx.fVpid)
2208 {
2209 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2210 {
2211 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2212 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2213 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2214 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2215 else
2216 {
2217 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2218 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2219 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2220 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2221 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2222 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2223 pVM->hm.s.vmx.fVpid = false;
2224 }
2225 }
2226 else
2227 {
2228 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2229 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2230 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2231 pVM->hm.s.vmx.fVpid = false;
2232 }
2233 }
2234
2235 /*
2236 * Setup the handler for flushing tagged-TLBs.
2237 */
2238 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2239 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2240 else if (pVM->hm.s.fNestedPaging)
2241 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2242 else if (pVM->hm.s.vmx.fVpid)
2243 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2244 else
2245 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2246 return VINF_SUCCESS;
2247}
2248
2249
2250/**
2251 * Sets up pin-based VM-execution controls in the VMCS.
2252 *
2253 * @returns VBox status code.
2254 * @param pVM Pointer to the VM.
2255 * @param pVCpu Pointer to the VMCPU.
2256 */
2257static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2258{
2259 AssertPtr(pVM);
2260 AssertPtr(pVCpu);
2261
2262 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2263 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2264
2265 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2266 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2267
2268 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2269 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2270
2271 /* Enable the VMX preemption timer. */
2272 if (pVM->hm.s.vmx.fUsePreemptTimer)
2273 {
2274 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2275 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2276 }
2277
2278 if ((val & zap) != val)
2279 {
2280 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2281 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2282 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2283 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2284 }
2285
2286 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2287 AssertRCReturn(rc, rc);
2288
2289 pVCpu->hm.s.vmx.u32PinCtls = val;
2290 return rc;
2291}
2292
2293
2294/**
2295 * Sets up processor-based VM-execution controls in the VMCS.
2296 *
2297 * @returns VBox status code.
2298 * @param pVM Pointer to the VM.
2299 * @param pVMCPU Pointer to the VMCPU.
2300 */
2301static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2302{
2303 AssertPtr(pVM);
2304 AssertPtr(pVCpu);
2305
2306 int rc = VERR_INTERNAL_ERROR_5;
2307 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2308 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2309
2310 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2311 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2312 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2313 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2314 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2315 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2316 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2317
2318 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2319 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2320 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2321 {
2322 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2323 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2324 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2325 }
2326
2327 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2328 if (!pVM->hm.s.fNestedPaging)
2329 {
2330 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2331 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2332 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2333 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2334 }
2335
2336 /* Use TPR shadowing if supported by the CPU. */
2337 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2338 {
2339 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2340 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2341 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2342 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2343 AssertRCReturn(rc, rc);
2344
2345 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2346 /* CR8 writes cause a VM-exit based on TPR threshold. */
2347 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2348 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2349 }
2350 else
2351 {
2352 /*
2353 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2354 * Set this control only for 64-bit guests.
2355 */
2356 if (pVM->hm.s.fAllow64BitGuests)
2357 {
2358 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2359 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2360 }
2361 }
2362
2363 /* Use MSR-bitmaps if supported by the CPU. */
2364 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2365 {
2366 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2367
2368 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2369 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2370 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2371 AssertRCReturn(rc, rc);
2372
2373 /*
2374 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2375 * automatically using dedicated fields in the VMCS.
2376 */
2377 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2378 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2379 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2380 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2381 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2382
2383#if HC_ARCH_BITS == 64
2384 /*
2385 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2386 */
2387 if (pVM->hm.s.fAllow64BitGuests)
2388 {
2389 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2390 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2391 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2392 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2393 }
2394#endif
2395 }
2396
2397 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2398 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2399 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2400
2401 if ((val & zap) != val)
2402 {
2403 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2404 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2405 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2406 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2407 }
2408
2409 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2410 AssertRCReturn(rc, rc);
2411
2412 pVCpu->hm.s.vmx.u32ProcCtls = val;
2413
2414 /*
2415 * Secondary processor-based VM-execution controls.
2416 */
2417 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2418 {
2419 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2420 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2421
2422 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2423 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2424
2425 if (pVM->hm.s.fNestedPaging)
2426 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2427 else
2428 {
2429 /*
2430 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2431 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2432 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2433 */
2434 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2435 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2436 }
2437
2438 if (pVM->hm.s.vmx.fVpid)
2439 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2440
2441 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2442 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2443
2444 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2445 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2446 * done dynamically. */
2447 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2448 {
2449 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2450 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2451 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2452 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2453 AssertRCReturn(rc, rc);
2454 }
2455
2456 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2457 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2458
2459 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2460 && pVM->hm.s.vmx.cPleGapTicks
2461 && pVM->hm.s.vmx.cPleWindowTicks)
2462 {
2463 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2464
2465 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2466 AssertRCReturn(rc, rc);
2467
2468 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2469 AssertRCReturn(rc, rc);
2470 }
2471
2472 if ((val & zap) != val)
2473 {
2474 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2475 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2476 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2477 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2478 }
2479
2480 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2481 AssertRCReturn(rc, rc);
2482
2483 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2484 }
2485 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2486 {
2487 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2488 "available\n"));
2489 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2490 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2491 }
2492
2493 return VINF_SUCCESS;
2494}
2495
2496
2497/**
2498 * Sets up miscellaneous (everything other than Pin & Processor-based
2499 * VM-execution) control fields in the VMCS.
2500 *
2501 * @returns VBox status code.
2502 * @param pVM Pointer to the VM.
2503 * @param pVCpu Pointer to the VMCPU.
2504 */
2505static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2506{
2507 NOREF(pVM);
2508 AssertPtr(pVM);
2509 AssertPtr(pVCpu);
2510
2511 int rc = VERR_GENERAL_FAILURE;
2512
2513 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2514#if 0
2515 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2516 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2517 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2518
2519 /*
2520 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2521 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2522 * We thus use the exception bitmap to control it rather than use both.
2523 */
2524 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2525 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2526
2527 /** @todo Explore possibility of using IO-bitmaps. */
2528 /* All IO & IOIO instructions cause VM-exits. */
2529 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2530 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2531
2532 /* Initialize the MSR-bitmap area. */
2533 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2535 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2536#endif
2537
2538 /* Setup MSR auto-load/store area. */
2539 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2540 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2541 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2542 AssertRCReturn(rc, rc);
2543 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2544 AssertRCReturn(rc, rc);
2545
2546 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2547 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2548 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2549 AssertRCReturn(rc, rc);
2550
2551 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2552 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2553 AssertRCReturn(rc, rc);
2554
2555 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2556#if 0
2557 /* Setup debug controls */
2558 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2559 AssertRCReturn(rc, rc);
2560 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2561 AssertRCReturn(rc, rc);
2562#endif
2563
2564 return rc;
2565}
2566
2567
2568/**
2569 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2570 *
2571 * @returns VBox status code.
2572 * @param pVM Pointer to the VM.
2573 * @param pVCpu Pointer to the VMCPU.
2574 */
2575static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2576{
2577 AssertPtr(pVM);
2578 AssertPtr(pVCpu);
2579
2580 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2581
2582 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2583
2584 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2585 if (!pVM->hm.s.fNestedPaging)
2586 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2587
2588 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2589 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2590 AssertRCReturn(rc, rc);
2591 return rc;
2592}
2593
2594
2595/**
2596 * Sets up the initial guest-state mask. The guest-state mask is consulted
2597 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2598 * for the nested virtualization case (as it would cause a VM-exit).
2599 *
2600 * @param pVCpu Pointer to the VMCPU.
2601 */
2602static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2603{
2604 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2605 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2606 return VINF_SUCCESS;
2607}
2608
2609
2610/**
2611 * Does per-VM VT-x initialization.
2612 *
2613 * @returns VBox status code.
2614 * @param pVM Pointer to the VM.
2615 */
2616VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2617{
2618 LogFlowFunc(("pVM=%p\n", pVM));
2619
2620 int rc = hmR0VmxStructsAlloc(pVM);
2621 if (RT_FAILURE(rc))
2622 {
2623 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2624 return rc;
2625 }
2626
2627 return VINF_SUCCESS;
2628}
2629
2630
2631/**
2632 * Does per-VM VT-x termination.
2633 *
2634 * @returns VBox status code.
2635 * @param pVM Pointer to the VM.
2636 */
2637VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2638{
2639 LogFlowFunc(("pVM=%p\n", pVM));
2640
2641#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2642 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2643 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2644#endif
2645 hmR0VmxStructsFree(pVM);
2646 return VINF_SUCCESS;
2647}
2648
2649
2650/**
2651 * Sets up the VM for execution under VT-x.
2652 * This function is only called once per-VM during initialization.
2653 *
2654 * @returns VBox status code.
2655 * @param pVM Pointer to the VM.
2656 */
2657VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2658{
2659 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2660 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2661
2662 LogFlowFunc(("pVM=%p\n", pVM));
2663
2664 /*
2665 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2666 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2667 */
2668 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2669 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2670 || !pVM->hm.s.vmx.pRealModeTSS))
2671 {
2672 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2673 return VERR_INTERNAL_ERROR;
2674 }
2675
2676 /* Initialize these always, see hmR3InitFinalizeR0().*/
2677 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2678 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2679
2680 /* Setup the tagged-TLB flush handlers. */
2681 int rc = hmR0VmxSetupTaggedTlb(pVM);
2682 if (RT_FAILURE(rc))
2683 {
2684 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2685 return rc;
2686 }
2687
2688 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2689 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2690#if HC_ARCH_BITS == 64
2691 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2692 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2693 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2694 {
2695 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2696 }
2697#endif
2698
2699 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2700 {
2701 PVMCPU pVCpu = &pVM->aCpus[i];
2702 AssertPtr(pVCpu);
2703 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2704
2705 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2706 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2707
2708 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2709 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2710 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2711
2712 /* Set revision dword at the beginning of the VMCS structure. */
2713 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2714
2715 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2716 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2717 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2718 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2719
2720 /* Load this VMCS as the current VMCS. */
2721 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2722 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2723 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2724
2725 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2726 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2727 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2728
2729 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2730 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2731 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2732
2733 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2734 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2735 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2736
2737 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2738 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2739 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2740
2741 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2742 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2743 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2744
2745#if HC_ARCH_BITS == 32
2746 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2747 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2748 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2749#endif
2750
2751 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2752 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2753 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2754 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2755
2756 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2757
2758 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2759 }
2760
2761 return VINF_SUCCESS;
2762}
2763
2764
2765/**
2766 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2767 * the VMCS.
2768 *
2769 * @returns VBox status code.
2770 * @param pVM Pointer to the VM.
2771 * @param pVCpu Pointer to the VMCPU.
2772 */
2773DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2774{
2775 NOREF(pVM); NOREF(pVCpu);
2776
2777 RTCCUINTREG uReg = ASMGetCR0();
2778 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2779 AssertRCReturn(rc, rc);
2780
2781 uReg = ASMGetCR3();
2782 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2783 AssertRCReturn(rc, rc);
2784
2785 uReg = ASMGetCR4();
2786 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2787 AssertRCReturn(rc, rc);
2788 return rc;
2789}
2790
2791
2792#if HC_ARCH_BITS == 64
2793/**
2794 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2795 * requirements. See hmR0VmxSaveHostSegmentRegs().
2796 */
2797# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2798 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2799 { \
2800 bool fValidSelector = true; \
2801 if ((selValue) & X86_SEL_LDT) \
2802 { \
2803 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2804 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2805 } \
2806 if (fValidSelector) \
2807 { \
2808 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2809 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2810 } \
2811 (selValue) = 0; \
2812 }
2813#endif
2814
2815
2816/**
2817 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2818 * the host-state area in the VMCS.
2819 *
2820 * @returns VBox status code.
2821 * @param pVM Pointer to the VM.
2822 * @param pVCpu Pointer to the VMCPU.
2823 */
2824DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2825{
2826 int rc = VERR_INTERNAL_ERROR_5;
2827
2828#if HC_ARCH_BITS == 64
2829 /*
2830 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2831 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2832 */
2833 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2834 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2835#endif
2836
2837 /*
2838 * Host DS, ES, FS and GS segment registers.
2839 */
2840#if HC_ARCH_BITS == 64
2841 RTSEL uSelDS = ASMGetDS();
2842 RTSEL uSelES = ASMGetES();
2843 RTSEL uSelFS = ASMGetFS();
2844 RTSEL uSelGS = ASMGetGS();
2845#else
2846 RTSEL uSelDS = 0;
2847 RTSEL uSelES = 0;
2848 RTSEL uSelFS = 0;
2849 RTSEL uSelGS = 0;
2850#endif
2851
2852 /* Recalculate which host-state bits need to be manually restored. */
2853 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2854
2855 /*
2856 * Host CS and SS segment registers.
2857 */
2858 RTSEL uSelCS = ASMGetCS();
2859 RTSEL uSelSS = ASMGetSS();
2860
2861 /*
2862 * Host TR segment register.
2863 */
2864 RTSEL uSelTR = ASMGetTR();
2865
2866#if HC_ARCH_BITS == 64
2867 /*
2868 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2869 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2870 */
2871 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2872 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2873 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2874 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2875# undef VMXLOCAL_ADJUST_HOST_SEG
2876#endif
2877
2878 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2879 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2880 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2881 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2882 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2883 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2884 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2885 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2886 Assert(uSelCS);
2887 Assert(uSelTR);
2888
2889 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2890#if 0
2891 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2892 Assert(uSelSS != 0);
2893#endif
2894
2895 /* Write these host selector fields into the host-state area in the VMCS. */
2896 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2897 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2898#if HC_ARCH_BITS == 64
2899 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2900 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2901 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2902 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2903#else
2904 NOREF(uSelDS);
2905 NOREF(uSelES);
2906 NOREF(uSelFS);
2907 NOREF(uSelGS);
2908#endif
2909 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2910
2911 /*
2912 * Host GDTR and IDTR.
2913 */
2914 RTGDTR Gdtr;
2915 RTIDTR Idtr;
2916 RT_ZERO(Gdtr);
2917 RT_ZERO(Idtr);
2918 ASMGetGDTR(&Gdtr);
2919 ASMGetIDTR(&Idtr);
2920 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2921 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2922
2923#if HC_ARCH_BITS == 64
2924 /*
2925 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2926 * maximum limit (0xffff) on every VM-exit.
2927 */
2928 if (Gdtr.cbGdt != 0xffff)
2929 {
2930 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2931 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2932 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2933 }
2934
2935 /*
2936 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
2937 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
2938 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
2939 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
2940 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
2941 * hosts where we are pretty sure it won't cause trouble.
2942 */
2943# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2944 if (Idtr.cbIdt < 0x0fff)
2945# else
2946 if (Idtr.cbIdt != 0xffff)
2947# endif
2948 {
2949 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2950 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2951 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2952 }
2953#endif
2954
2955 /*
2956 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2957 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2958 */
2959 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
2960 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
2961 VERR_VMX_INVALID_HOST_STATE);
2962
2963 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2964#if HC_ARCH_BITS == 64
2965 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
2966
2967 /*
2968 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2969 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2970 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2971 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2972 *
2973 * [1] See Intel spec. 3.5 "System Descriptor Types".
2974 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2975 */
2976 Assert(pDesc->System.u4Type == 11);
2977 if ( pDesc->System.u16LimitLow != 0x67
2978 || pDesc->System.u4LimitHigh)
2979 {
2980 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2981 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
2982 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
2983 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
2984 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2985
2986 /* Store the GDTR here as we need it while restoring TR. */
2987 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2988 }
2989#else
2990 NOREF(pVM);
2991 uintptr_t uTRBase = X86DESC_BASE(pDesc);
2992#endif
2993 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2994 AssertRCReturn(rc, rc);
2995
2996 /*
2997 * Host FS base and GS base.
2998 */
2999#if HC_ARCH_BITS == 64
3000 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3001 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3002 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3003 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3004
3005 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3006 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3007 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3008 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3009 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3010#endif
3011 return rc;
3012}
3013
3014
3015/**
3016 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3017 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3018 * the host after every successful VM-exit.
3019 *
3020 * @returns VBox status code.
3021 * @param pVM Pointer to the VM.
3022 * @param pVCpu Pointer to the VMCPU.
3023 *
3024 * @remarks No-long-jump zone!!!
3025 */
3026DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3027{
3028 NOREF(pVM);
3029
3030 AssertPtr(pVCpu);
3031 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3032
3033 int rc = VINF_SUCCESS;
3034#if HC_ARCH_BITS == 64
3035 if (pVM->hm.s.fAllow64BitGuests)
3036 hmR0VmxLazySaveHostMsrs(pVCpu);
3037#endif
3038
3039 /*
3040 * Host Sysenter MSRs.
3041 */
3042 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3043 AssertRCReturn(rc, rc);
3044#if HC_ARCH_BITS == 32
3045 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3046 AssertRCReturn(rc, rc);
3047 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3048#else
3049 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3050 AssertRCReturn(rc, rc);
3051 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3052#endif
3053 AssertRCReturn(rc, rc);
3054
3055 /*
3056 * Host EFER MSR.
3057 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3058 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3059 */
3060 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3061 {
3062 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3063 AssertRCReturn(rc, rc);
3064 }
3065
3066 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3067 * hmR0VmxLoadGuestExitCtls() !! */
3068
3069 return rc;
3070}
3071
3072
3073/**
3074 * Figures out if we need to swap the EFER MSR which is
3075 * particularly expensive.
3076 *
3077 * We check all relevant bits. For now, that's everything
3078 * besides LMA/LME, as these two bits are handled by VM-entry,
3079 * see hmR0VmxLoadGuestExitCtls() and
3080 * hmR0VMxLoadGuestEntryCtls().
3081 *
3082 * @returns true if we need to load guest EFER, false otherwise.
3083 * @param pVCpu Pointer to the VMCPU.
3084 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3085 * out-of-sync. Make sure to update the required fields
3086 * before using them.
3087 *
3088 * @remarks Requires EFER, CR4.
3089 * @remarks No-long-jump zone!!!
3090 */
3091static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3092{
3093#ifdef HMVMX_ALWAYS_SWAP_EFER
3094 return true;
3095#endif
3096
3097#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3098 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3099 if (CPUMIsGuestInLongMode(pVCpu))
3100 return false;
3101#endif
3102
3103 PVM pVM = pVCpu->CTX_SUFF(pVM);
3104 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3105 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3106
3107 /*
3108 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3109 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3110 */
3111 if ( CPUMIsGuestInLongMode(pVCpu)
3112 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3113 {
3114 return true;
3115 }
3116
3117 /*
3118 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3119 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3120 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3121 */
3122 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3123 && (pMixedCtx->cr0 & X86_CR0_PG)
3124 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3125 {
3126 /* Assert that host is PAE capable. */
3127 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3128 return true;
3129 }
3130
3131 /** @todo Check the latest Intel spec. for any other bits,
3132 * like SMEP/SMAP? */
3133 return false;
3134}
3135
3136
3137/**
3138 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3139 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3140 * controls".
3141 *
3142 * @returns VBox status code.
3143 * @param pVCpu Pointer to the VMCPU.
3144 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3145 * out-of-sync. Make sure to update the required fields
3146 * before using them.
3147 *
3148 * @remarks Requires EFER.
3149 * @remarks No-long-jump zone!!!
3150 */
3151DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3152{
3153 int rc = VINF_SUCCESS;
3154 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3155 {
3156 PVM pVM = pVCpu->CTX_SUFF(pVM);
3157 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3158 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3159
3160 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3161 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3162
3163 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3164 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3165 {
3166 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3167 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3168 }
3169 else
3170 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3171
3172 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3173 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3174 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3175 {
3176 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3177 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3178 }
3179
3180 /*
3181 * The following should -not- be set (since we're not in SMM mode):
3182 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3183 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3184 */
3185
3186 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3187 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3188
3189 if ((val & zap) != val)
3190 {
3191 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3192 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3193 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3194 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3195 }
3196
3197 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3198 AssertRCReturn(rc, rc);
3199
3200 pVCpu->hm.s.vmx.u32EntryCtls = val;
3201 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3202 }
3203 return rc;
3204}
3205
3206
3207/**
3208 * Sets up the VM-exit controls in the VMCS.
3209 *
3210 * @returns VBox status code.
3211 * @param pVM Pointer to the VM.
3212 * @param pVCpu Pointer to the VMCPU.
3213 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3214 * out-of-sync. Make sure to update the required fields
3215 * before using them.
3216 *
3217 * @remarks Requires EFER.
3218 */
3219DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3220{
3221 NOREF(pMixedCtx);
3222
3223 int rc = VINF_SUCCESS;
3224 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3225 {
3226 PVM pVM = pVCpu->CTX_SUFF(pVM);
3227 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3228 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3229
3230 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3231 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3232
3233 /*
3234 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3235 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3236 */
3237#if HC_ARCH_BITS == 64
3238 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3239 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3240#else
3241 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3242 {
3243 /* The switcher returns to long mode, EFER is managed by the switcher. */
3244 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3245 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3246 }
3247 else
3248 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3249#endif
3250
3251 /* If the newer VMCS fields for managing EFER exists, use it. */
3252 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3253 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3254 {
3255 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3256 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3257 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3258 }
3259
3260 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3261 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3262
3263 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3264 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3265 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3266
3267 if ( pVM->hm.s.vmx.fUsePreemptTimer
3268 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3269 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3270
3271 if ((val & zap) != val)
3272 {
3273 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3274 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3275 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3276 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3277 }
3278
3279 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3280 AssertRCReturn(rc, rc);
3281
3282 pVCpu->hm.s.vmx.u32ExitCtls = val;
3283 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3284 }
3285 return rc;
3286}
3287
3288
3289/**
3290 * Loads the guest APIC and related state.
3291 *
3292 * @returns VBox status code.
3293 * @param pVM Pointer to the VM.
3294 * @param pVCpu Pointer to the VMCPU.
3295 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3296 * out-of-sync. Make sure to update the required fields
3297 * before using them.
3298 */
3299DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3300{
3301 NOREF(pMixedCtx);
3302
3303 int rc = VINF_SUCCESS;
3304 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3305 {
3306 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3307 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3308 {
3309 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3310
3311 bool fPendingIntr = false;
3312 uint8_t u8Tpr = 0;
3313 uint8_t u8PendingIntr = 0;
3314 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3315 AssertRCReturn(rc, rc);
3316
3317 /*
3318 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3319 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3320 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3321 * the interrupt when we VM-exit for other reasons.
3322 */
3323 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3324 uint32_t u32TprThreshold = 0;
3325 if (fPendingIntr)
3326 {
3327 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3328 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3329 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3330 if (u8PendingPriority <= u8TprPriority)
3331 u32TprThreshold = u8PendingPriority;
3332 else
3333 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3334 }
3335 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3336
3337 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3338 AssertRCReturn(rc, rc);
3339 }
3340
3341 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3342 }
3343 return rc;
3344}
3345
3346
3347/**
3348 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3349 *
3350 * @returns Guest's interruptibility-state.
3351 * @param pVCpu Pointer to the VMCPU.
3352 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3353 * out-of-sync. Make sure to update the required fields
3354 * before using them.
3355 *
3356 * @remarks No-long-jump zone!!!
3357 */
3358DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3359{
3360 /*
3361 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3362 */
3363 uint32_t uIntrState = 0;
3364 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3365 {
3366 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3367 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3368 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3369 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3370 {
3371 if (pMixedCtx->eflags.Bits.u1IF)
3372 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3373 else
3374 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3375 }
3376 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3377 }
3378
3379 /*
3380 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3381 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3382 * setting this would block host-NMIs and IRET will not clear the blocking.
3383 *
3384 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3385 */
3386 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3387 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3388 {
3389 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3390 }
3391
3392 return uIntrState;
3393}
3394
3395
3396/**
3397 * Loads the guest's interruptibility-state into the guest-state area in the
3398 * VMCS.
3399 *
3400 * @returns VBox status code.
3401 * @param pVCpu Pointer to the VMCPU.
3402 * @param uIntrState The interruptibility-state to set.
3403 */
3404static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3405{
3406 NOREF(pVCpu);
3407 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3408 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3409 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3410 AssertRCReturn(rc, rc);
3411 return rc;
3412}
3413
3414
3415/**
3416 * Loads the exception intercepts required for guest execution in the VMCS.
3417 *
3418 * @returns VBox status code.
3419 * @param pVCpu Pointer to the VMCPU.
3420 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3421 * out-of-sync. Make sure to update the required fields
3422 * before using them.
3423 */
3424static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3425{
3426 NOREF(pMixedCtx);
3427 int rc = VINF_SUCCESS;
3428 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3429 {
3430 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3431 if (pVCpu->hm.s.fGIMTrapXcptUD)
3432 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3433 else
3434 {
3435#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3436 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3437#endif
3438 }
3439
3440 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3441 AssertRCReturn(rc, rc);
3442
3443 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3444 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3445 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3446 }
3447 return rc;
3448}
3449
3450
3451/**
3452 * Loads the guest's RIP into the guest-state area in the VMCS.
3453 *
3454 * @returns VBox status code.
3455 * @param pVCpu Pointer to the VMCPU.
3456 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3457 * out-of-sync. Make sure to update the required fields
3458 * before using them.
3459 *
3460 * @remarks No-long-jump zone!!!
3461 */
3462static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3463{
3464 int rc = VINF_SUCCESS;
3465 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3466 {
3467 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3468 AssertRCReturn(rc, rc);
3469
3470 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3471 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3472 HMCPU_CF_VALUE(pVCpu)));
3473 }
3474 return rc;
3475}
3476
3477
3478/**
3479 * Loads the guest's RSP into the guest-state area in the VMCS.
3480 *
3481 * @returns VBox status code.
3482 * @param pVCpu Pointer to the VMCPU.
3483 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3484 * out-of-sync. Make sure to update the required fields
3485 * before using them.
3486 *
3487 * @remarks No-long-jump zone!!!
3488 */
3489static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3490{
3491 int rc = VINF_SUCCESS;
3492 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3493 {
3494 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3495 AssertRCReturn(rc, rc);
3496
3497 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3498 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3499 }
3500 return rc;
3501}
3502
3503
3504/**
3505 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3506 *
3507 * @returns VBox status code.
3508 * @param pVCpu Pointer to the VMCPU.
3509 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3510 * out-of-sync. Make sure to update the required fields
3511 * before using them.
3512 *
3513 * @remarks No-long-jump zone!!!
3514 */
3515static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3516{
3517 int rc = VINF_SUCCESS;
3518 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3519 {
3520 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3521 Let us assert it as such and use 32-bit VMWRITE. */
3522 Assert(!(pMixedCtx->rflags.u64 >> 32));
3523 X86EFLAGS Eflags = pMixedCtx->eflags;
3524 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3525 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3526 * These will never be cleared/set, unless some other part of the VMM
3527 * code is buggy - in which case we're better of finding and fixing
3528 * those bugs than hiding them. */
3529 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3530 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3531 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3532 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3533
3534 /*
3535 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3536 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3537 */
3538 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3539 {
3540 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3541 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3542 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3543 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3544 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3545 }
3546
3547 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3548 AssertRCReturn(rc, rc);
3549
3550 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3551 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3552 }
3553 return rc;
3554}
3555
3556
3557/**
3558 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3559 *
3560 * @returns VBox status code.
3561 * @param pVCpu Pointer to the VMCPU.
3562 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3563 * out-of-sync. Make sure to update the required fields
3564 * before using them.
3565 *
3566 * @remarks No-long-jump zone!!!
3567 */
3568DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3569{
3570 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3571 AssertRCReturn(rc, rc);
3572 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3573 AssertRCReturn(rc, rc);
3574 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3575 AssertRCReturn(rc, rc);
3576 return rc;
3577}
3578
3579
3580/**
3581 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3582 * CR0 is partially shared with the host and we have to consider the FPU bits.
3583 *
3584 * @returns VBox status code.
3585 * @param pVM Pointer to the VM.
3586 * @param pVCpu Pointer to the VMCPU.
3587 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3588 * out-of-sync. Make sure to update the required fields
3589 * before using them.
3590 *
3591 * @remarks No-long-jump zone!!!
3592 */
3593static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3594{
3595 /*
3596 * Guest CR0.
3597 * Guest FPU.
3598 */
3599 int rc = VINF_SUCCESS;
3600 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3601 {
3602 Assert(!(pMixedCtx->cr0 >> 32));
3603 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3604 PVM pVM = pVCpu->CTX_SUFF(pVM);
3605
3606 /* The guest's view (read access) of its CR0 is unblemished. */
3607 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3608 AssertRCReturn(rc, rc);
3609 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3610
3611 /* Setup VT-x's view of the guest CR0. */
3612 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3613 if (pVM->hm.s.fNestedPaging)
3614 {
3615 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3616 {
3617 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3618 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3619 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3620 }
3621 else
3622 {
3623 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3624 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3625 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3626 }
3627
3628 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3629 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3630 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3631
3632 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3633 AssertRCReturn(rc, rc);
3634 }
3635 else
3636 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3637
3638 /*
3639 * Guest FPU bits.
3640 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3641 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3642 */
3643 u32GuestCR0 |= X86_CR0_NE;
3644 bool fInterceptNM = false;
3645 if (CPUMIsGuestFPUStateActive(pVCpu))
3646 {
3647 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3648 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3649 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3650 }
3651 else
3652 {
3653 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3654 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3655 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3656 }
3657
3658 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3659 bool fInterceptMF = false;
3660 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3661 fInterceptMF = true;
3662
3663 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3664 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3665 {
3666 Assert(PDMVmmDevHeapIsEnabled(pVM));
3667 Assert(pVM->hm.s.vmx.pRealModeTSS);
3668 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3669 fInterceptNM = true;
3670 fInterceptMF = true;
3671 }
3672 else
3673 {
3674 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3675 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3676 }
3677 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3678
3679 if (fInterceptNM)
3680 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3681 else
3682 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3683
3684 if (fInterceptMF)
3685 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3686 else
3687 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3688
3689 /* Additional intercepts for debugging, define these yourself explicitly. */
3690#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3691 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3692 | RT_BIT(X86_XCPT_BP)
3693 | RT_BIT(X86_XCPT_DB)
3694 | RT_BIT(X86_XCPT_DE)
3695 | RT_BIT(X86_XCPT_NM)
3696 | RT_BIT(X86_XCPT_TS)
3697 | RT_BIT(X86_XCPT_UD)
3698 | RT_BIT(X86_XCPT_NP)
3699 | RT_BIT(X86_XCPT_SS)
3700 | RT_BIT(X86_XCPT_GP)
3701 | RT_BIT(X86_XCPT_PF)
3702 | RT_BIT(X86_XCPT_MF)
3703 ;
3704#elif defined(HMVMX_ALWAYS_TRAP_PF)
3705 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3706#endif
3707
3708 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3709
3710 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3711 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3712 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3713 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3714 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3715 else
3716 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3717
3718 u32GuestCR0 |= uSetCR0;
3719 u32GuestCR0 &= uZapCR0;
3720 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3721
3722 /* Write VT-x's view of the guest CR0 into the VMCS. */
3723 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3724 AssertRCReturn(rc, rc);
3725 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3726 uZapCR0));
3727
3728 /*
3729 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3730 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3731 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3732 */
3733 uint32_t u32CR0Mask = 0;
3734 u32CR0Mask = X86_CR0_PE
3735 | X86_CR0_NE
3736 | X86_CR0_WP
3737 | X86_CR0_PG
3738 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3739 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3740 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3741
3742 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3743 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3744 * and @bugref{6944}. */
3745#if 0
3746 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3747 u32CR0Mask &= ~X86_CR0_PE;
3748#endif
3749 if (pVM->hm.s.fNestedPaging)
3750 u32CR0Mask &= ~X86_CR0_WP;
3751
3752 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3753 if (fInterceptNM)
3754 {
3755 u32CR0Mask |= X86_CR0_TS
3756 | X86_CR0_MP;
3757 }
3758
3759 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3760 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3761 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3762 AssertRCReturn(rc, rc);
3763 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3764
3765 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3766 }
3767 return rc;
3768}
3769
3770
3771/**
3772 * Loads the guest control registers (CR3, CR4) into the guest-state area
3773 * in the VMCS.
3774 *
3775 * @returns VBox status code.
3776 * @param pVM Pointer to the VM.
3777 * @param pVCpu Pointer to the VMCPU.
3778 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3779 * out-of-sync. Make sure to update the required fields
3780 * before using them.
3781 *
3782 * @remarks No-long-jump zone!!!
3783 */
3784static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3785{
3786 int rc = VINF_SUCCESS;
3787 PVM pVM = pVCpu->CTX_SUFF(pVM);
3788
3789 /*
3790 * Guest CR2.
3791 * It's always loaded in the assembler code. Nothing to do here.
3792 */
3793
3794 /*
3795 * Guest CR3.
3796 */
3797 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3798 {
3799 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3800 if (pVM->hm.s.fNestedPaging)
3801 {
3802 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3803
3804 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3805 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3806 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3807 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3808
3809 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3810 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3811 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3812
3813 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3814 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3815 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3816 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3817 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3818 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3819 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3820
3821 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3822 AssertRCReturn(rc, rc);
3823 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3824
3825 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3826 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3827 {
3828 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3829 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3830 {
3831 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3832 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3833 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3834 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3835 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3836 }
3837
3838 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3839 have Unrestricted Execution to handle the guest when it's not using paging. */
3840 GCPhysGuestCR3 = pMixedCtx->cr3;
3841 }
3842 else
3843 {
3844 /*
3845 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3846 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3847 * EPT takes care of translating it to host-physical addresses.
3848 */
3849 RTGCPHYS GCPhys;
3850 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3851 Assert(PDMVmmDevHeapIsEnabled(pVM));
3852
3853 /* We obtain it here every time as the guest could have relocated this PCI region. */
3854 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3855 AssertRCReturn(rc, rc);
3856
3857 GCPhysGuestCR3 = GCPhys;
3858 }
3859
3860 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3861 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3862 }
3863 else
3864 {
3865 /* Non-nested paging case, just use the hypervisor's CR3. */
3866 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3867
3868 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3869 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3870 }
3871 AssertRCReturn(rc, rc);
3872
3873 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3874 }
3875
3876 /*
3877 * Guest CR4.
3878 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3879 */
3880 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3881 {
3882 Assert(!(pMixedCtx->cr4 >> 32));
3883 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3884
3885 /* The guest's view of its CR4 is unblemished. */
3886 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3887 AssertRCReturn(rc, rc);
3888 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3889
3890 /* Setup VT-x's view of the guest CR4. */
3891 /*
3892 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3893 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3894 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3895 */
3896 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3897 {
3898 Assert(pVM->hm.s.vmx.pRealModeTSS);
3899 Assert(PDMVmmDevHeapIsEnabled(pVM));
3900 u32GuestCR4 &= ~X86_CR4_VME;
3901 }
3902
3903 if (pVM->hm.s.fNestedPaging)
3904 {
3905 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3906 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3907 {
3908 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3909 u32GuestCR4 |= X86_CR4_PSE;
3910 /* Our identity mapping is a 32-bit page directory. */
3911 u32GuestCR4 &= ~X86_CR4_PAE;
3912 }
3913 /* else use guest CR4.*/
3914 }
3915 else
3916 {
3917 /*
3918 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3919 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3920 */
3921 switch (pVCpu->hm.s.enmShadowMode)
3922 {
3923 case PGMMODE_REAL: /* Real-mode. */
3924 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3925 case PGMMODE_32_BIT: /* 32-bit paging. */
3926 {
3927 u32GuestCR4 &= ~X86_CR4_PAE;
3928 break;
3929 }
3930
3931 case PGMMODE_PAE: /* PAE paging. */
3932 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3933 {
3934 u32GuestCR4 |= X86_CR4_PAE;
3935 break;
3936 }
3937
3938 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3939 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3940#ifdef VBOX_ENABLE_64_BITS_GUESTS
3941 break;
3942#endif
3943 default:
3944 AssertFailed();
3945 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3946 }
3947 }
3948
3949 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3950 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3951 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3952 u32GuestCR4 |= uSetCR4;
3953 u32GuestCR4 &= uZapCR4;
3954
3955 /* Write VT-x's view of the guest CR4 into the VMCS. */
3956 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
3957 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3958 AssertRCReturn(rc, rc);
3959
3960 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3961 uint32_t u32CR4Mask = X86_CR4_VME
3962 | X86_CR4_PAE
3963 | X86_CR4_PGE
3964 | X86_CR4_PSE
3965 | X86_CR4_VMXE;
3966 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
3967 u32CR4Mask |= X86_CR4_OSXSAVE;
3968 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3969 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3970 AssertRCReturn(rc, rc);
3971
3972 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
3973 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
3974
3975 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3976 }
3977 return rc;
3978}
3979
3980
3981/**
3982 * Loads the guest debug registers into the guest-state area in the VMCS.
3983 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
3984 *
3985 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3986 *
3987 * @returns VBox status code.
3988 * @param pVCpu Pointer to the VMCPU.
3989 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3990 * out-of-sync. Make sure to update the required fields
3991 * before using them.
3992 *
3993 * @remarks No-long-jump zone!!!
3994 */
3995static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3996{
3997 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3998 return VINF_SUCCESS;
3999
4000#ifdef VBOX_STRICT
4001 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4002 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4003 {
4004 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4005 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4006 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4007 }
4008#endif
4009
4010 int rc;
4011 PVM pVM = pVCpu->CTX_SUFF(pVM);
4012 bool fInterceptDB = false;
4013 bool fInterceptMovDRx = false;
4014 if ( pVCpu->hm.s.fSingleInstruction
4015 || DBGFIsStepping(pVCpu))
4016 {
4017 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4018 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4019 {
4020 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4021 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4022 AssertRCReturn(rc, rc);
4023 Assert(fInterceptDB == false);
4024 }
4025 else
4026 {
4027 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4028 pVCpu->hm.s.fClearTrapFlag = true;
4029 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4030 fInterceptDB = true;
4031 }
4032 }
4033
4034 if ( fInterceptDB
4035 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4036 {
4037 /*
4038 * Use the combined guest and host DRx values found in the hypervisor
4039 * register set because the debugger has breakpoints active or someone
4040 * is single stepping on the host side without a monitor trap flag.
4041 *
4042 * Note! DBGF expects a clean DR6 state before executing guest code.
4043 */
4044#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4045 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4046 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4047 {
4048 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4049 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4050 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4051 }
4052 else
4053#endif
4054 if (!CPUMIsHyperDebugStateActive(pVCpu))
4055 {
4056 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4057 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4058 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4059 }
4060
4061 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4062 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4063 AssertRCReturn(rc, rc);
4064
4065 pVCpu->hm.s.fUsingHyperDR7 = true;
4066 fInterceptDB = true;
4067 fInterceptMovDRx = true;
4068 }
4069 else
4070 {
4071 /*
4072 * If the guest has enabled debug registers, we need to load them prior to
4073 * executing guest code so they'll trigger at the right time.
4074 */
4075 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4076 {
4077#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4078 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4079 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4080 {
4081 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4082 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4083 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4084 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4085 }
4086 else
4087#endif
4088 if (!CPUMIsGuestDebugStateActive(pVCpu))
4089 {
4090 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4091 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4092 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4093 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4094 }
4095 Assert(!fInterceptDB);
4096 Assert(!fInterceptMovDRx);
4097 }
4098 /*
4099 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4100 * must intercept #DB in order to maintain a correct DR6 guest value.
4101 */
4102#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4103 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4104 && !CPUMIsGuestDebugStateActive(pVCpu))
4105#else
4106 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4107#endif
4108 {
4109 fInterceptMovDRx = true;
4110 fInterceptDB = true;
4111 }
4112
4113 /* Update guest DR7. */
4114 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4115 AssertRCReturn(rc, rc);
4116
4117 pVCpu->hm.s.fUsingHyperDR7 = false;
4118 }
4119
4120 /*
4121 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4122 */
4123 if ( fInterceptDB
4124 || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4125 {
4126 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4127 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
4128 }
4129 else
4130 {
4131#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4132 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4133 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
4134#endif
4135 }
4136
4137 /*
4138 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4139 */
4140 if (fInterceptMovDRx)
4141 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4142 else
4143 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4144 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4145 AssertRCReturn(rc, rc);
4146
4147 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4148 return VINF_SUCCESS;
4149}
4150
4151
4152#ifdef VBOX_STRICT
4153/**
4154 * Strict function to validate segment registers.
4155 *
4156 * @remarks ASSUMES CR0 is up to date.
4157 */
4158static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4159{
4160 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4161 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4162 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4163 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4164 && ( !CPUMIsGuestInRealModeEx(pCtx)
4165 && !CPUMIsGuestInV86ModeEx(pCtx)))
4166 {
4167 /* Protected mode checks */
4168 /* CS */
4169 Assert(pCtx->cs.Attr.n.u1Present);
4170 Assert(!(pCtx->cs.Attr.u & 0xf00));
4171 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4172 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4173 || !(pCtx->cs.Attr.n.u1Granularity));
4174 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4175 || (pCtx->cs.Attr.n.u1Granularity));
4176 /* CS cannot be loaded with NULL in protected mode. */
4177 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4178 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4179 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4180 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4181 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4182 else
4183 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4184 /* SS */
4185 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4186 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4187 if ( !(pCtx->cr0 & X86_CR0_PE)
4188 || pCtx->cs.Attr.n.u4Type == 3)
4189 {
4190 Assert(!pCtx->ss.Attr.n.u2Dpl);
4191 }
4192 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4193 {
4194 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4195 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4196 Assert(pCtx->ss.Attr.n.u1Present);
4197 Assert(!(pCtx->ss.Attr.u & 0xf00));
4198 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4199 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4200 || !(pCtx->ss.Attr.n.u1Granularity));
4201 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4202 || (pCtx->ss.Attr.n.u1Granularity));
4203 }
4204 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4205 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4206 {
4207 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4208 Assert(pCtx->ds.Attr.n.u1Present);
4209 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4210 Assert(!(pCtx->ds.Attr.u & 0xf00));
4211 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4212 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4213 || !(pCtx->ds.Attr.n.u1Granularity));
4214 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4215 || (pCtx->ds.Attr.n.u1Granularity));
4216 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4217 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4218 }
4219 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4220 {
4221 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4222 Assert(pCtx->es.Attr.n.u1Present);
4223 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4224 Assert(!(pCtx->es.Attr.u & 0xf00));
4225 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4226 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4227 || !(pCtx->es.Attr.n.u1Granularity));
4228 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4229 || (pCtx->es.Attr.n.u1Granularity));
4230 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4231 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4232 }
4233 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4234 {
4235 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4236 Assert(pCtx->fs.Attr.n.u1Present);
4237 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4238 Assert(!(pCtx->fs.Attr.u & 0xf00));
4239 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4240 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4241 || !(pCtx->fs.Attr.n.u1Granularity));
4242 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4243 || (pCtx->fs.Attr.n.u1Granularity));
4244 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4245 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4246 }
4247 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4248 {
4249 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4250 Assert(pCtx->gs.Attr.n.u1Present);
4251 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4252 Assert(!(pCtx->gs.Attr.u & 0xf00));
4253 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4254 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4255 || !(pCtx->gs.Attr.n.u1Granularity));
4256 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4257 || (pCtx->gs.Attr.n.u1Granularity));
4258 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4259 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4260 }
4261 /* 64-bit capable CPUs. */
4262# if HC_ARCH_BITS == 64
4263 Assert(!(pCtx->cs.u64Base >> 32));
4264 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4265 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4266 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4267# endif
4268 }
4269 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4270 || ( CPUMIsGuestInRealModeEx(pCtx)
4271 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4272 {
4273 /* Real and v86 mode checks. */
4274 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4275 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4276 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4277 {
4278 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4279 }
4280 else
4281 {
4282 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4283 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4284 }
4285
4286 /* CS */
4287 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4288 Assert(pCtx->cs.u32Limit == 0xffff);
4289 Assert(u32CSAttr == 0xf3);
4290 /* SS */
4291 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4292 Assert(pCtx->ss.u32Limit == 0xffff);
4293 Assert(u32SSAttr == 0xf3);
4294 /* DS */
4295 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4296 Assert(pCtx->ds.u32Limit == 0xffff);
4297 Assert(u32DSAttr == 0xf3);
4298 /* ES */
4299 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4300 Assert(pCtx->es.u32Limit == 0xffff);
4301 Assert(u32ESAttr == 0xf3);
4302 /* FS */
4303 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4304 Assert(pCtx->fs.u32Limit == 0xffff);
4305 Assert(u32FSAttr == 0xf3);
4306 /* GS */
4307 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4308 Assert(pCtx->gs.u32Limit == 0xffff);
4309 Assert(u32GSAttr == 0xf3);
4310 /* 64-bit capable CPUs. */
4311# if HC_ARCH_BITS == 64
4312 Assert(!(pCtx->cs.u64Base >> 32));
4313 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4314 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4315 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4316# endif
4317 }
4318}
4319#endif /* VBOX_STRICT */
4320
4321
4322/**
4323 * Writes a guest segment register into the guest-state area in the VMCS.
4324 *
4325 * @returns VBox status code.
4326 * @param pVCpu Pointer to the VMCPU.
4327 * @param idxSel Index of the selector in the VMCS.
4328 * @param idxLimit Index of the segment limit in the VMCS.
4329 * @param idxBase Index of the segment base in the VMCS.
4330 * @param idxAccess Index of the access rights of the segment in the VMCS.
4331 * @param pSelReg Pointer to the segment selector.
4332 *
4333 * @remarks No-long-jump zone!!!
4334 */
4335static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4336 uint32_t idxAccess, PCPUMSELREG pSelReg)
4337{
4338 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4339 AssertRCReturn(rc, rc);
4340 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4341 AssertRCReturn(rc, rc);
4342 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4343 AssertRCReturn(rc, rc);
4344
4345 uint32_t u32Access = pSelReg->Attr.u;
4346 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4347 {
4348 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4349 u32Access = 0xf3;
4350 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4351 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4352 }
4353 else
4354 {
4355 /*
4356 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4357 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4358 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4359 * loaded in protected-mode have their attribute as 0.
4360 */
4361 if (!u32Access)
4362 u32Access = X86DESCATTR_UNUSABLE;
4363 }
4364
4365 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4366 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4367 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4368
4369 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4370 AssertRCReturn(rc, rc);
4371 return rc;
4372}
4373
4374
4375/**
4376 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4377 * into the guest-state area in the VMCS.
4378 *
4379 * @returns VBox status code.
4380 * @param pVM Pointer to the VM.
4381 * @param pVCPU Pointer to the VMCPU.
4382 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4383 * out-of-sync. Make sure to update the required fields
4384 * before using them.
4385 *
4386 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4387 * @remarks No-long-jump zone!!!
4388 */
4389static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4390{
4391 int rc = VERR_INTERNAL_ERROR_5;
4392 PVM pVM = pVCpu->CTX_SUFF(pVM);
4393
4394 /*
4395 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4396 */
4397 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4398 {
4399 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4400 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4401 {
4402 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4403 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4404 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4405 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4406 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4407 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4408 }
4409
4410#ifdef VBOX_WITH_REM
4411 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4412 {
4413 Assert(pVM->hm.s.vmx.pRealModeTSS);
4414 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4415 if ( pVCpu->hm.s.vmx.fWasInRealMode
4416 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4417 {
4418 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4419 in real-mode (e.g. OpenBSD 4.0) */
4420 REMFlushTBs(pVM);
4421 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4422 pVCpu->hm.s.vmx.fWasInRealMode = false;
4423 }
4424 }
4425#endif
4426 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4427 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4428 AssertRCReturn(rc, rc);
4429 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4430 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4431 AssertRCReturn(rc, rc);
4432 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4433 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4434 AssertRCReturn(rc, rc);
4435 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4436 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4437 AssertRCReturn(rc, rc);
4438 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4439 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4440 AssertRCReturn(rc, rc);
4441 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4442 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4443 AssertRCReturn(rc, rc);
4444
4445#ifdef VBOX_STRICT
4446 /* Validate. */
4447 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4448#endif
4449
4450 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4451 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4452 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4453 }
4454
4455 /*
4456 * Guest TR.
4457 */
4458 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4459 {
4460 /*
4461 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4462 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4463 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4464 */
4465 uint16_t u16Sel = 0;
4466 uint32_t u32Limit = 0;
4467 uint64_t u64Base = 0;
4468 uint32_t u32AccessRights = 0;
4469
4470 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4471 {
4472 u16Sel = pMixedCtx->tr.Sel;
4473 u32Limit = pMixedCtx->tr.u32Limit;
4474 u64Base = pMixedCtx->tr.u64Base;
4475 u32AccessRights = pMixedCtx->tr.Attr.u;
4476 }
4477 else
4478 {
4479 Assert(pVM->hm.s.vmx.pRealModeTSS);
4480 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4481
4482 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4483 RTGCPHYS GCPhys;
4484 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4485 AssertRCReturn(rc, rc);
4486
4487 X86DESCATTR DescAttr;
4488 DescAttr.u = 0;
4489 DescAttr.n.u1Present = 1;
4490 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4491
4492 u16Sel = 0;
4493 u32Limit = HM_VTX_TSS_SIZE;
4494 u64Base = GCPhys; /* in real-mode phys = virt. */
4495 u32AccessRights = DescAttr.u;
4496 }
4497
4498 /* Validate. */
4499 Assert(!(u16Sel & RT_BIT(2)));
4500 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4501 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4502 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4503 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4504 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4505 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4506 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4507 Assert( (u32Limit & 0xfff) == 0xfff
4508 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4509 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4510 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4511
4512 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4513 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4514 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4515 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4516
4517 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4518 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4519 }
4520
4521 /*
4522 * Guest GDTR.
4523 */
4524 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4525 {
4526 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4527 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4528
4529 /* Validate. */
4530 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4531
4532 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4533 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4534 }
4535
4536 /*
4537 * Guest LDTR.
4538 */
4539 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4540 {
4541 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4542 uint32_t u32Access = 0;
4543 if (!pMixedCtx->ldtr.Attr.u)
4544 u32Access = X86DESCATTR_UNUSABLE;
4545 else
4546 u32Access = pMixedCtx->ldtr.Attr.u;
4547
4548 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4549 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4550 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4551 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4552
4553 /* Validate. */
4554 if (!(u32Access & X86DESCATTR_UNUSABLE))
4555 {
4556 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4557 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4558 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4559 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4560 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4561 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4562 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4563 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4564 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4565 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4566 }
4567
4568 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4569 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4570 }
4571
4572 /*
4573 * Guest IDTR.
4574 */
4575 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4576 {
4577 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4578 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4579
4580 /* Validate. */
4581 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4582
4583 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4584 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4585 }
4586
4587 return VINF_SUCCESS;
4588}
4589
4590
4591/**
4592 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4593 * areas.
4594 *
4595 * These MSRs will automatically be loaded to the host CPU on every successful
4596 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4597 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4598 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4599 *
4600 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4601 *
4602 * @returns VBox status code.
4603 * @param pVCpu Pointer to the VMCPU.
4604 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4605 * out-of-sync. Make sure to update the required fields
4606 * before using them.
4607 *
4608 * @remarks No-long-jump zone!!!
4609 */
4610static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4611{
4612 AssertPtr(pVCpu);
4613 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4614
4615 /*
4616 * MSRs that we use the auto-load/store MSR area in the VMCS.
4617 */
4618 PVM pVM = pVCpu->CTX_SUFF(pVM);
4619 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4620 {
4621 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4622#if HC_ARCH_BITS == 32
4623 if (pVM->hm.s.fAllow64BitGuests)
4624 {
4625 int rc = VINF_SUCCESS;
4626 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4627 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4628 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4629 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4630 AssertRCReturn(rc, rc);
4631#ifdef DEBUG
4632 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4633 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4634 {
4635 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4636 pMsr->u64Value));
4637 }
4638# endif
4639 }
4640#endif
4641 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4642 }
4643
4644 /*
4645 * Guest Sysenter MSRs.
4646 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4647 * VM-exits on WRMSRs for these MSRs.
4648 */
4649 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4650 {
4651 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4652 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4653 }
4654
4655 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4656 {
4657 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4658 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4659 }
4660
4661 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4662 {
4663 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4665 }
4666
4667 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4668 {
4669 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4670 {
4671 /*
4672 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4673 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4674 */
4675 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4676 {
4677 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4678 AssertRCReturn(rc,rc);
4679 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4680 }
4681 else
4682 {
4683 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4684 NULL /* pfAddedAndUpdated */);
4685 AssertRCReturn(rc, rc);
4686
4687 /* We need to intercept reads too, see @bugref{7386#c16}. */
4688 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4689 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4690 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4691 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4692 }
4693 }
4694 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4695 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4696 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4697 }
4698
4699 return VINF_SUCCESS;
4700}
4701
4702
4703/**
4704 * Loads the guest activity state into the guest-state area in the VMCS.
4705 *
4706 * @returns VBox status code.
4707 * @param pVCpu Pointer to the VMCPU.
4708 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4709 * out-of-sync. Make sure to update the required fields
4710 * before using them.
4711 *
4712 * @remarks No-long-jump zone!!!
4713 */
4714static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4715{
4716 NOREF(pCtx);
4717 /** @todo See if we can make use of other states, e.g.
4718 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4719 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4720 {
4721 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4722 AssertRCReturn(rc, rc);
4723
4724 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4725 }
4726 return VINF_SUCCESS;
4727}
4728
4729
4730/**
4731 * Sets up the appropriate function to run guest code.
4732 *
4733 * @returns VBox status code.
4734 * @param pVCpu Pointer to the VMCPU.
4735 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4736 * out-of-sync. Make sure to update the required fields
4737 * before using them.
4738 *
4739 * @remarks No-long-jump zone!!!
4740 */
4741static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4742{
4743 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4744 {
4745#ifndef VBOX_ENABLE_64_BITS_GUESTS
4746 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4747#endif
4748 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4749#if HC_ARCH_BITS == 32
4750 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4751 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4752 {
4753 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4754 {
4755 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4756 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4757 | HM_CHANGED_VMX_ENTRY_CTLS
4758 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4759 }
4760 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4761 }
4762#else
4763 /* 64-bit host. */
4764 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4765#endif
4766 }
4767 else
4768 {
4769 /* Guest is not in long mode, use the 32-bit handler. */
4770#if HC_ARCH_BITS == 32
4771 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4772 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4773 {
4774 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4775 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4776 | HM_CHANGED_VMX_ENTRY_CTLS
4777 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4778 }
4779#endif
4780 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4781 }
4782 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4783 return VINF_SUCCESS;
4784}
4785
4786
4787/**
4788 * Wrapper for running the guest code in VT-x.
4789 *
4790 * @returns VBox strict status code.
4791 * @param pVM Pointer to the VM.
4792 * @param pVCpu Pointer to the VMCPU.
4793 * @param pCtx Pointer to the guest-CPU context.
4794 *
4795 * @remarks No-long-jump zone!!!
4796 */
4797DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4798{
4799 /*
4800 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4801 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4802 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4803 */
4804 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4805 /** @todo Add stats for resume vs launch. */
4806#ifdef VBOX_WITH_KERNEL_USING_XMM
4807 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4808#else
4809 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4810#endif
4811}
4812
4813
4814/**
4815 * Reports world-switch error and dumps some useful debug info.
4816 *
4817 * @param pVM Pointer to the VM.
4818 * @param pVCpu Pointer to the VMCPU.
4819 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4820 * @param pCtx Pointer to the guest-CPU context.
4821 * @param pVmxTransient Pointer to the VMX transient structure (only
4822 * exitReason updated).
4823 */
4824static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4825{
4826 Assert(pVM);
4827 Assert(pVCpu);
4828 Assert(pCtx);
4829 Assert(pVmxTransient);
4830 HMVMX_ASSERT_PREEMPT_SAFE();
4831
4832 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4833 switch (rcVMRun)
4834 {
4835 case VERR_VMX_INVALID_VMXON_PTR:
4836 AssertFailed();
4837 break;
4838 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4839 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4840 {
4841 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4842 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4843 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4844 AssertRC(rc);
4845
4846 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4847 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4848 Cannot do it here as we may have been long preempted. */
4849
4850#ifdef VBOX_STRICT
4851 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4852 pVmxTransient->uExitReason));
4853 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4854 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4855 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4856 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4857 else
4858 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4859 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4860 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4861
4862 /* VMX control bits. */
4863 uint32_t u32Val;
4864 uint64_t u64Val;
4865 RTHCUINTREG uHCReg;
4866 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4867 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4868 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4869 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4870 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4871 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4872 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4873 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4874 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4875 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4876 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4877 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4878 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4879 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4880 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4881 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4882 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4883 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4884 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4885 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4886 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4887 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4888 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4889 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4890 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4891 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4892 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4893 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4894 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4895 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4896 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4897 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4898 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4899 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4900 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4901 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4902 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4903 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4904 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4905 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4906 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4907 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4908
4909 /* Guest bits. */
4910 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4911 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4912 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4913 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4914 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4915 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4916 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4917 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4918
4919 /* Host bits. */
4920 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4921 Log4(("Host CR0 %#RHr\n", uHCReg));
4922 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4923 Log4(("Host CR3 %#RHr\n", uHCReg));
4924 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4925 Log4(("Host CR4 %#RHr\n", uHCReg));
4926
4927 RTGDTR HostGdtr;
4928 PCX86DESCHC pDesc;
4929 ASMGetGDTR(&HostGdtr);
4930 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4931 Log4(("Host CS %#08x\n", u32Val));
4932 if (u32Val < HostGdtr.cbGdt)
4933 {
4934 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4935 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4936 }
4937
4938 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4939 Log4(("Host DS %#08x\n", u32Val));
4940 if (u32Val < HostGdtr.cbGdt)
4941 {
4942 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4943 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4944 }
4945
4946 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4947 Log4(("Host ES %#08x\n", u32Val));
4948 if (u32Val < HostGdtr.cbGdt)
4949 {
4950 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4951 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4952 }
4953
4954 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4955 Log4(("Host FS %#08x\n", u32Val));
4956 if (u32Val < HostGdtr.cbGdt)
4957 {
4958 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4959 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4960 }
4961
4962 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4963 Log4(("Host GS %#08x\n", u32Val));
4964 if (u32Val < HostGdtr.cbGdt)
4965 {
4966 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4967 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4968 }
4969
4970 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4971 Log4(("Host SS %#08x\n", u32Val));
4972 if (u32Val < HostGdtr.cbGdt)
4973 {
4974 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4975 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4976 }
4977
4978 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4979 Log4(("Host TR %#08x\n", u32Val));
4980 if (u32Val < HostGdtr.cbGdt)
4981 {
4982 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4983 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4984 }
4985
4986 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4987 Log4(("Host TR Base %#RHv\n", uHCReg));
4988 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4989 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4990 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4991 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4992 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4993 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4994 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4995 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4996 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4997 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4998 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4999 Log4(("Host RSP %#RHv\n", uHCReg));
5000 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5001 Log4(("Host RIP %#RHv\n", uHCReg));
5002# if HC_ARCH_BITS == 64
5003 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5004 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5005 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5006 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5007 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5008 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5009# endif
5010#endif /* VBOX_STRICT */
5011 break;
5012 }
5013
5014 default:
5015 /* Impossible */
5016 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5017 break;
5018 }
5019 NOREF(pVM); NOREF(pCtx);
5020}
5021
5022
5023#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5024#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5025# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5026#endif
5027#ifdef VBOX_STRICT
5028static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5029{
5030 switch (idxField)
5031 {
5032 case VMX_VMCS_GUEST_RIP:
5033 case VMX_VMCS_GUEST_RSP:
5034 case VMX_VMCS_GUEST_SYSENTER_EIP:
5035 case VMX_VMCS_GUEST_SYSENTER_ESP:
5036 case VMX_VMCS_GUEST_GDTR_BASE:
5037 case VMX_VMCS_GUEST_IDTR_BASE:
5038 case VMX_VMCS_GUEST_CS_BASE:
5039 case VMX_VMCS_GUEST_DS_BASE:
5040 case VMX_VMCS_GUEST_ES_BASE:
5041 case VMX_VMCS_GUEST_FS_BASE:
5042 case VMX_VMCS_GUEST_GS_BASE:
5043 case VMX_VMCS_GUEST_SS_BASE:
5044 case VMX_VMCS_GUEST_LDTR_BASE:
5045 case VMX_VMCS_GUEST_TR_BASE:
5046 case VMX_VMCS_GUEST_CR3:
5047 return true;
5048 }
5049 return false;
5050}
5051
5052static bool hmR0VmxIsValidReadField(uint32_t idxField)
5053{
5054 switch (idxField)
5055 {
5056 /* Read-only fields. */
5057 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5058 return true;
5059 }
5060 /* Remaining readable fields should also be writable. */
5061 return hmR0VmxIsValidWriteField(idxField);
5062}
5063#endif /* VBOX_STRICT */
5064
5065
5066/**
5067 * Executes the specified handler in 64-bit mode.
5068 *
5069 * @returns VBox status code.
5070 * @param pVM Pointer to the VM.
5071 * @param pVCpu Pointer to the VMCPU.
5072 * @param pCtx Pointer to the guest CPU context.
5073 * @param enmOp The operation to perform.
5074 * @param cParams Number of parameters.
5075 * @param paParam Array of 32-bit parameters.
5076 */
5077VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5078 uint32_t cParams, uint32_t *paParam)
5079{
5080 NOREF(pCtx);
5081
5082 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5083 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5084 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5085 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5086
5087#ifdef VBOX_STRICT
5088 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5089 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5090
5091 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5092 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5093#endif
5094
5095 /* Disable interrupts. */
5096 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5097
5098#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5099 RTCPUID idHostCpu = RTMpCpuId();
5100 CPUMR0SetLApic(pVCpu, idHostCpu);
5101#endif
5102
5103 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5104 RTHCPHYS HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5105
5106 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5107 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5108
5109 /* Leave VMX Root Mode. */
5110 VMXDisable();
5111
5112 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5113
5114 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5115 CPUMSetHyperEIP(pVCpu, enmOp);
5116 for (int i = (int)cParams - 1; i >= 0; i--)
5117 CPUMPushHyper(pVCpu, paParam[i]);
5118
5119 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5120
5121 /* Call the switcher. */
5122 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5123 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5124
5125 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5126 /* Make sure the VMX instructions don't cause #UD faults. */
5127 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5128
5129 /* Re-enter VMX Root Mode */
5130 int rc2 = VMXEnable(HCPhysCpuPage);
5131 if (RT_FAILURE(rc2))
5132 {
5133 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5134 ASMSetFlags(fOldEFlags);
5135 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5136 return rc2;
5137 }
5138
5139 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5140 AssertRC(rc2);
5141 Assert(!(ASMGetFlags() & X86_EFL_IF));
5142 ASMSetFlags(fOldEFlags);
5143 return rc;
5144}
5145
5146
5147/**
5148 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5149 * supporting 64-bit guests.
5150 *
5151 * @returns VBox status code.
5152 * @param fResume Whether to VMLAUNCH or VMRESUME.
5153 * @param pCtx Pointer to the guest-CPU context.
5154 * @param pCache Pointer to the VMCS cache.
5155 * @param pVM Pointer to the VM.
5156 * @param pVCpu Pointer to the VMCPU.
5157 */
5158DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5159{
5160 NOREF(fResume);
5161
5162 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5163 RTHCPHYS HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5164
5165#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5166 pCache->uPos = 1;
5167 pCache->interPD = PGMGetInterPaeCR3(pVM);
5168 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5169#endif
5170
5171#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5172 pCache->TestIn.HCPhysCpuPage = 0;
5173 pCache->TestIn.HCPhysVmcs = 0;
5174 pCache->TestIn.pCache = 0;
5175 pCache->TestOut.HCPhysVmcs = 0;
5176 pCache->TestOut.pCache = 0;
5177 pCache->TestOut.pCtx = 0;
5178 pCache->TestOut.eflags = 0;
5179#endif
5180
5181 uint32_t aParam[10];
5182 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5183 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5184 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5185 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5186 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5187 aParam[5] = 0;
5188 aParam[6] = VM_RC_ADDR(pVM, pVM);
5189 aParam[7] = 0;
5190 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5191 aParam[9] = 0;
5192
5193#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5194 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5195 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5196#endif
5197 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5198
5199#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5200 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5201 Assert(pCtx->dr[4] == 10);
5202 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5203#endif
5204
5205#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5206 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5207 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5208 pVCpu->hm.s.vmx.HCPhysVmcs));
5209 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5210 pCache->TestOut.HCPhysVmcs));
5211 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5212 pCache->TestOut.pCache));
5213 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5214 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5215 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5216 pCache->TestOut.pCtx));
5217 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5218#endif
5219 return rc;
5220}
5221
5222
5223/**
5224 * Initialize the VMCS-Read cache.
5225 *
5226 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5227 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5228 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5229 * (those that have a 32-bit FULL & HIGH part).
5230 *
5231 * @returns VBox status code.
5232 * @param pVM Pointer to the VM.
5233 * @param pVCpu Pointer to the VMCPU.
5234 */
5235static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5236{
5237#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5238{ \
5239 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5240 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5241 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5242 ++cReadFields; \
5243}
5244
5245 AssertPtr(pVM);
5246 AssertPtr(pVCpu);
5247 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5248 uint32_t cReadFields = 0;
5249
5250 /*
5251 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5252 * and serve to indicate exceptions to the rules.
5253 */
5254
5255 /* Guest-natural selector base fields. */
5256#if 0
5257 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5258 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5259 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5260#endif
5261 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5262 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5263 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5264 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5265 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5266 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5267 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5268 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5269 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5270 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5271 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5272 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5273#if 0
5274 /* Unused natural width guest-state fields. */
5275 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5276 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5277#endif
5278 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5279 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5280
5281 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5282#if 0
5283 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5284 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5285 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5286 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5287 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5288 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5289 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5290 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5291 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5292#endif
5293
5294 /* Natural width guest-state fields. */
5295 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5296#if 0
5297 /* Currently unused field. */
5298 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5299#endif
5300
5301 if (pVM->hm.s.fNestedPaging)
5302 {
5303 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5304 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5305 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5306 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5307 }
5308 else
5309 {
5310 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5311 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5312 }
5313
5314#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5315 return VINF_SUCCESS;
5316}
5317
5318
5319/**
5320 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5321 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5322 * darwin, running 64-bit guests).
5323 *
5324 * @returns VBox status code.
5325 * @param pVCpu Pointer to the VMCPU.
5326 * @param idxField The VMCS field encoding.
5327 * @param u64Val 16, 32 or 64-bit value.
5328 */
5329VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5330{
5331 int rc;
5332 switch (idxField)
5333 {
5334 /*
5335 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5336 */
5337 /* 64-bit Control fields. */
5338 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5339 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5340 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5341 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5342 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5343 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5344 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5345 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5346 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5347 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5348 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5349 case VMX_VMCS64_CTRL_EPTP_FULL:
5350 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5351 /* 64-bit Guest-state fields. */
5352 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5353 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5354 case VMX_VMCS64_GUEST_PAT_FULL:
5355 case VMX_VMCS64_GUEST_EFER_FULL:
5356 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5357 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5358 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5359 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5360 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5361 /* 64-bit Host-state fields. */
5362 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5363 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5364 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5365 {
5366 rc = VMXWriteVmcs32(idxField, u64Val);
5367 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5368 break;
5369 }
5370
5371 /*
5372 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5373 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5374 */
5375 /* Natural-width Guest-state fields. */
5376 case VMX_VMCS_GUEST_CR3:
5377 case VMX_VMCS_GUEST_ES_BASE:
5378 case VMX_VMCS_GUEST_CS_BASE:
5379 case VMX_VMCS_GUEST_SS_BASE:
5380 case VMX_VMCS_GUEST_DS_BASE:
5381 case VMX_VMCS_GUEST_FS_BASE:
5382 case VMX_VMCS_GUEST_GS_BASE:
5383 case VMX_VMCS_GUEST_LDTR_BASE:
5384 case VMX_VMCS_GUEST_TR_BASE:
5385 case VMX_VMCS_GUEST_GDTR_BASE:
5386 case VMX_VMCS_GUEST_IDTR_BASE:
5387 case VMX_VMCS_GUEST_RSP:
5388 case VMX_VMCS_GUEST_RIP:
5389 case VMX_VMCS_GUEST_SYSENTER_ESP:
5390 case VMX_VMCS_GUEST_SYSENTER_EIP:
5391 {
5392 if (!(u64Val >> 32))
5393 {
5394 /* If this field is 64-bit, VT-x will zero out the top bits. */
5395 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5396 }
5397 else
5398 {
5399 /* Assert that only the 32->64 switcher case should ever come here. */
5400 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5401 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5402 }
5403 break;
5404 }
5405
5406 default:
5407 {
5408 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5409 rc = VERR_INVALID_PARAMETER;
5410 break;
5411 }
5412 }
5413 AssertRCReturn(rc, rc);
5414 return rc;
5415}
5416
5417
5418/**
5419 * Queue up a VMWRITE by using the VMCS write cache.
5420 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5421 *
5422 * @param pVCpu Pointer to the VMCPU.
5423 * @param idxField The VMCS field encoding.
5424 * @param u64Val 16, 32 or 64-bit value.
5425 */
5426VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5427{
5428 AssertPtr(pVCpu);
5429 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5430
5431 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5432 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5433
5434 /* Make sure there are no duplicates. */
5435 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5436 {
5437 if (pCache->Write.aField[i] == idxField)
5438 {
5439 pCache->Write.aFieldVal[i] = u64Val;
5440 return VINF_SUCCESS;
5441 }
5442 }
5443
5444 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5445 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5446 pCache->Write.cValidEntries++;
5447 return VINF_SUCCESS;
5448}
5449#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5450
5451
5452/**
5453 * Sets up the usage of TSC-offsetting and updates the VMCS.
5454 *
5455 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5456 * VMX preemption timer.
5457 *
5458 * @returns VBox status code.
5459 * @param pVM Pointer to the cross context VM structure.
5460 * @param pVCpu Pointer to the VMCPU.
5461 *
5462 * @remarks No-long-jump zone!!!
5463 */
5464static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5465{
5466 int rc;
5467 bool fOffsettedTsc;
5468 bool fParavirtTsc;
5469 if (pVM->hm.s.vmx.fUsePreemptTimer)
5470 {
5471 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5472 &fOffsettedTsc, &fParavirtTsc);
5473
5474 /* Make sure the returned values have sane upper and lower boundaries. */
5475 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5476 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5477 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5478 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5479
5480 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5481 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5482 }
5483 else
5484 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5485
5486 /** @todo later optimize this to be done elsewhere and not before every
5487 * VM-entry. */
5488 if (fParavirtTsc)
5489 {
5490 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5491 information before every VM-entry, hence disable it for performance sake. */
5492#if 0
5493 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5494 AssertRC(rc);
5495#endif
5496 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5497 }
5498
5499 if (fOffsettedTsc)
5500 {
5501 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5502 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5503
5504 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5505 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5506 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5507 }
5508 else
5509 {
5510 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5511 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5512 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5513 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5514 }
5515}
5516
5517
5518/**
5519 * Determines if an exception is a contributory exception.
5520 *
5521 * Contributory exceptions are ones which can cause double-faults unless the
5522 * original exception was a benign exception. Page-fault is intentionally not
5523 * included here as it's a conditional contributory exception.
5524 *
5525 * @returns true if the exception is contributory, false otherwise.
5526 * @param uVector The exception vector.
5527 */
5528DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5529{
5530 switch (uVector)
5531 {
5532 case X86_XCPT_GP:
5533 case X86_XCPT_SS:
5534 case X86_XCPT_NP:
5535 case X86_XCPT_TS:
5536 case X86_XCPT_DE:
5537 return true;
5538 default:
5539 break;
5540 }
5541 return false;
5542}
5543
5544
5545/**
5546 * Sets an event as a pending event to be injected into the guest.
5547 *
5548 * @param pVCpu Pointer to the VMCPU.
5549 * @param u32IntInfo The VM-entry interruption-information field.
5550 * @param cbInstr The VM-entry instruction length in bytes (for software
5551 * interrupts, exceptions and privileged software
5552 * exceptions).
5553 * @param u32ErrCode The VM-entry exception error code.
5554 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5555 * page-fault.
5556 *
5557 * @remarks Statistics counter assumes this is a guest event being injected or
5558 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5559 * always incremented.
5560 */
5561DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5562 RTGCUINTPTR GCPtrFaultAddress)
5563{
5564 Assert(!pVCpu->hm.s.Event.fPending);
5565 pVCpu->hm.s.Event.fPending = true;
5566 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5567 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5568 pVCpu->hm.s.Event.cbInstr = cbInstr;
5569 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5570
5571 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5572}
5573
5574
5575/**
5576 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5577 *
5578 * @param pVCpu Pointer to the VMCPU.
5579 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5580 * out-of-sync. Make sure to update the required fields
5581 * before using them.
5582 */
5583DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5584{
5585 NOREF(pMixedCtx);
5586 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5587 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5588 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5589 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5590}
5591
5592
5593/**
5594 * Handle a condition that occurred while delivering an event through the guest
5595 * IDT.
5596 *
5597 * @returns VBox status code (informational error codes included).
5598 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5599 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5600 * continue execution of the guest which will delivery the #DF.
5601 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5602 *
5603 * @param pVCpu Pointer to the VMCPU.
5604 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5605 * out-of-sync. Make sure to update the required fields
5606 * before using them.
5607 * @param pVmxTransient Pointer to the VMX transient structure.
5608 *
5609 * @remarks No-long-jump zone!!!
5610 */
5611static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5612{
5613 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5614
5615 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5616 AssertRCReturn(rc, rc);
5617 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5618 AssertRCReturn(rc, rc);
5619
5620 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5621 {
5622 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5623 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5624
5625 typedef enum
5626 {
5627 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5628 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5629 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5630 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5631 } VMXREFLECTXCPT;
5632
5633 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5634 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5635 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5636 {
5637 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5638 {
5639 enmReflect = VMXREFLECTXCPT_XCPT;
5640#ifdef VBOX_STRICT
5641 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5642 && uExitVector == X86_XCPT_PF)
5643 {
5644 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5645 }
5646#endif
5647 if ( uExitVector == X86_XCPT_PF
5648 && uIdtVector == X86_XCPT_PF)
5649 {
5650 pVmxTransient->fVectoringDoublePF = true;
5651 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5652 }
5653 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5654 && hmR0VmxIsContributoryXcpt(uExitVector)
5655 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5656 || uIdtVector == X86_XCPT_PF))
5657 {
5658 enmReflect = VMXREFLECTXCPT_DF;
5659 }
5660 else if (uIdtVector == X86_XCPT_DF)
5661 enmReflect = VMXREFLECTXCPT_TF;
5662 }
5663 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5664 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5665 {
5666 /*
5667 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5668 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5669 */
5670 enmReflect = VMXREFLECTXCPT_XCPT;
5671
5672 if (uExitVector == X86_XCPT_PF)
5673 {
5674 pVmxTransient->fVectoringPF = true;
5675 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5676 }
5677 }
5678 }
5679 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5680 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5681 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5682 {
5683 /*
5684 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5685 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5686 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5687 */
5688 enmReflect = VMXREFLECTXCPT_XCPT;
5689 }
5690
5691 /*
5692 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5693 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5694 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5695 *
5696 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5697 */
5698 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5699 && enmReflect == VMXREFLECTXCPT_XCPT
5700 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5701 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5702 {
5703 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5704 }
5705
5706 switch (enmReflect)
5707 {
5708 case VMXREFLECTXCPT_XCPT:
5709 {
5710 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5711 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5712 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5713
5714 uint32_t u32ErrCode = 0;
5715 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5716 {
5717 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5718 AssertRCReturn(rc, rc);
5719 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5720 }
5721
5722 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5723 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5724 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5725 rc = VINF_SUCCESS;
5726 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5727 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5728
5729 break;
5730 }
5731
5732 case VMXREFLECTXCPT_DF:
5733 {
5734 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5735 rc = VINF_HM_DOUBLE_FAULT;
5736 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5737 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5738
5739 break;
5740 }
5741
5742 case VMXREFLECTXCPT_TF:
5743 {
5744 rc = VINF_EM_RESET;
5745 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5746 uExitVector));
5747 break;
5748 }
5749
5750 default:
5751 Assert(rc == VINF_SUCCESS);
5752 break;
5753 }
5754 }
5755 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5756 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5757 && uExitVector != X86_XCPT_DF
5758 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5759 {
5760 /*
5761 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5762 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5763 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5764 */
5765 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5766 {
5767 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5768 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5769 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5770 }
5771 }
5772
5773 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5774 return rc;
5775}
5776
5777
5778/**
5779 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5780 *
5781 * @returns VBox status code.
5782 * @param pVCpu Pointer to the VMCPU.
5783 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5784 * out-of-sync. Make sure to update the required fields
5785 * before using them.
5786 *
5787 * @remarks No-long-jump zone!!!
5788 */
5789static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5790{
5791 NOREF(pMixedCtx);
5792
5793 /*
5794 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5795 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5796 */
5797 VMMRZCallRing3Disable(pVCpu);
5798 HM_DISABLE_PREEMPT();
5799
5800 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5801 {
5802 uint32_t uVal = 0;
5803 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5804 AssertRCReturn(rc, rc);
5805
5806 uint32_t uShadow = 0;
5807 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5808 AssertRCReturn(rc, rc);
5809
5810 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5811 CPUMSetGuestCR0(pVCpu, uVal);
5812 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5813 }
5814
5815 HM_RESTORE_PREEMPT();
5816 VMMRZCallRing3Enable(pVCpu);
5817 return VINF_SUCCESS;
5818}
5819
5820
5821/**
5822 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5823 *
5824 * @returns VBox status code.
5825 * @param pVCpu Pointer to the VMCPU.
5826 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5827 * out-of-sync. Make sure to update the required fields
5828 * before using them.
5829 *
5830 * @remarks No-long-jump zone!!!
5831 */
5832static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5833{
5834 NOREF(pMixedCtx);
5835
5836 int rc = VINF_SUCCESS;
5837 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5838 {
5839 uint32_t uVal = 0;
5840 uint32_t uShadow = 0;
5841 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5842 AssertRCReturn(rc, rc);
5843 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5844 AssertRCReturn(rc, rc);
5845
5846 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5847 CPUMSetGuestCR4(pVCpu, uVal);
5848 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5849 }
5850 return rc;
5851}
5852
5853
5854/**
5855 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5856 *
5857 * @returns VBox status code.
5858 * @param pVCpu Pointer to the VMCPU.
5859 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5860 * out-of-sync. Make sure to update the required fields
5861 * before using them.
5862 *
5863 * @remarks No-long-jump zone!!!
5864 */
5865static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5866{
5867 int rc = VINF_SUCCESS;
5868 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5869 {
5870 uint64_t u64Val = 0;
5871 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5872 AssertRCReturn(rc, rc);
5873
5874 pMixedCtx->rip = u64Val;
5875 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5876 }
5877 return rc;
5878}
5879
5880
5881/**
5882 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5883 *
5884 * @returns VBox status code.
5885 * @param pVCpu Pointer to the VMCPU.
5886 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5887 * out-of-sync. Make sure to update the required fields
5888 * before using them.
5889 *
5890 * @remarks No-long-jump zone!!!
5891 */
5892static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5893{
5894 int rc = VINF_SUCCESS;
5895 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5896 {
5897 uint64_t u64Val = 0;
5898 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5899 AssertRCReturn(rc, rc);
5900
5901 pMixedCtx->rsp = u64Val;
5902 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5903 }
5904 return rc;
5905}
5906
5907
5908/**
5909 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5910 *
5911 * @returns VBox status code.
5912 * @param pVCpu Pointer to the VMCPU.
5913 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5914 * out-of-sync. Make sure to update the required fields
5915 * before using them.
5916 *
5917 * @remarks No-long-jump zone!!!
5918 */
5919static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5920{
5921 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5922 {
5923 uint32_t uVal = 0;
5924 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5925 AssertRCReturn(rc, rc);
5926
5927 pMixedCtx->eflags.u32 = uVal;
5928 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5929 {
5930 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5931 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5932
5933 pMixedCtx->eflags.Bits.u1VM = 0;
5934 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5935 }
5936
5937 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5938 }
5939 return VINF_SUCCESS;
5940}
5941
5942
5943/**
5944 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5945 * guest-CPU context.
5946 */
5947DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5948{
5949 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5950 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5951 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5952 return rc;
5953}
5954
5955
5956/**
5957 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5958 * from the guest-state area in the VMCS.
5959 *
5960 * @param pVCpu Pointer to the VMCPU.
5961 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5962 * out-of-sync. Make sure to update the required fields
5963 * before using them.
5964 *
5965 * @remarks No-long-jump zone!!!
5966 */
5967static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5968{
5969 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
5970 {
5971 uint32_t uIntrState = 0;
5972 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5973 AssertRC(rc);
5974
5975 if (!uIntrState)
5976 {
5977 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5978 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5979
5980 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5981 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5982 }
5983 else
5984 {
5985 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
5986 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
5987 {
5988 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5989 AssertRC(rc);
5990 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5991 AssertRC(rc);
5992
5993 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5994 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5995 }
5996 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5997 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5998
5999 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6000 {
6001 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6002 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6003 }
6004 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6005 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6006 }
6007
6008 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6009 }
6010}
6011
6012
6013/**
6014 * Saves the guest's activity state.
6015 *
6016 * @returns VBox status code.
6017 * @param pVCpu Pointer to the VMCPU.
6018 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6019 * out-of-sync. Make sure to update the required fields
6020 * before using them.
6021 *
6022 * @remarks No-long-jump zone!!!
6023 */
6024static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6025{
6026 NOREF(pMixedCtx);
6027 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6028 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6029 return VINF_SUCCESS;
6030}
6031
6032
6033/**
6034 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6035 * the current VMCS into the guest-CPU context.
6036 *
6037 * @returns VBox status code.
6038 * @param pVCpu Pointer to the VMCPU.
6039 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6040 * out-of-sync. Make sure to update the required fields
6041 * before using them.
6042 *
6043 * @remarks No-long-jump zone!!!
6044 */
6045static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6046{
6047 int rc = VINF_SUCCESS;
6048 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6049 {
6050 uint32_t u32Val = 0;
6051 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6052 pMixedCtx->SysEnter.cs = u32Val;
6053 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6054 }
6055
6056 uint64_t u64Val = 0;
6057 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6058 {
6059 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6060 pMixedCtx->SysEnter.eip = u64Val;
6061 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6062 }
6063 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6064 {
6065 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6066 pMixedCtx->SysEnter.esp = u64Val;
6067 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6068 }
6069 return rc;
6070}
6071
6072
6073/**
6074 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6075 * the CPU back into the guest-CPU context.
6076 *
6077 * @returns VBox status code.
6078 * @param pVCpu Pointer to the VMCPU.
6079 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6080 * out-of-sync. Make sure to update the required fields
6081 * before using them.
6082 *
6083 * @remarks No-long-jump zone!!!
6084 */
6085static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6086{
6087#if HC_ARCH_BITS == 64
6088 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6089 {
6090 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6091 VMMRZCallRing3Disable(pVCpu);
6092 HM_DISABLE_PREEMPT();
6093
6094 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6095 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6096 {
6097 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6098 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6099 }
6100
6101 HM_RESTORE_PREEMPT();
6102 VMMRZCallRing3Enable(pVCpu);
6103 }
6104 else
6105 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6106#else
6107 NOREF(pMixedCtx);
6108 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6109#endif
6110
6111 return VINF_SUCCESS;
6112}
6113
6114
6115/**
6116 * Saves the auto load/store'd guest MSRs from the current VMCS into
6117 * the guest-CPU context.
6118 *
6119 * @returns VBox status code.
6120 * @param pVCpu Pointer to the VMCPU.
6121 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6122 * out-of-sync. Make sure to update the required fields
6123 * before using them.
6124 *
6125 * @remarks No-long-jump zone!!!
6126 */
6127static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6128{
6129 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6130 return VINF_SUCCESS;
6131
6132 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6133 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6134 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6135 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6136 {
6137 switch (pMsr->u32Msr)
6138 {
6139 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6140 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6141 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6142 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6143 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6144 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6145 break;
6146
6147 default:
6148 {
6149 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6150 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6151 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6152 }
6153 }
6154 }
6155
6156 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6157 return VINF_SUCCESS;
6158}
6159
6160
6161/**
6162 * Saves the guest control registers from the current VMCS into the guest-CPU
6163 * context.
6164 *
6165 * @returns VBox status code.
6166 * @param pVCpu Pointer to the VMCPU.
6167 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6168 * out-of-sync. Make sure to update the required fields
6169 * before using them.
6170 *
6171 * @remarks No-long-jump zone!!!
6172 */
6173static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6174{
6175 /* Guest CR0. Guest FPU. */
6176 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6177 AssertRCReturn(rc, rc);
6178
6179 /* Guest CR4. */
6180 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6181 AssertRCReturn(rc, rc);
6182
6183 /* Guest CR2 - updated always during the world-switch or in #PF. */
6184 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6185 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6186 {
6187 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6188 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6189
6190 PVM pVM = pVCpu->CTX_SUFF(pVM);
6191 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6192 || ( pVM->hm.s.fNestedPaging
6193 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6194 {
6195 uint64_t u64Val = 0;
6196 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6197 if (pMixedCtx->cr3 != u64Val)
6198 {
6199 CPUMSetGuestCR3(pVCpu, u64Val);
6200 if (VMMRZCallRing3IsEnabled(pVCpu))
6201 {
6202 PGMUpdateCR3(pVCpu, u64Val);
6203 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6204 }
6205 else
6206 {
6207 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6208 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6209 }
6210 }
6211
6212 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6213 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6214 {
6215 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6216 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6217 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6218 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6219
6220 if (VMMRZCallRing3IsEnabled(pVCpu))
6221 {
6222 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6223 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6224 }
6225 else
6226 {
6227 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6228 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6229 }
6230 }
6231 }
6232
6233 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6234 }
6235
6236 /*
6237 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6238 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6239 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6240 *
6241 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6242 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6243 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6244 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6245 *
6246 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6247 */
6248 if (VMMRZCallRing3IsEnabled(pVCpu))
6249 {
6250 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6251 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6252
6253 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6254 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6255
6256 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6257 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6258 }
6259
6260 return rc;
6261}
6262
6263
6264/**
6265 * Reads a guest segment register from the current VMCS into the guest-CPU
6266 * context.
6267 *
6268 * @returns VBox status code.
6269 * @param pVCpu Pointer to the VMCPU.
6270 * @param idxSel Index of the selector in the VMCS.
6271 * @param idxLimit Index of the segment limit in the VMCS.
6272 * @param idxBase Index of the segment base in the VMCS.
6273 * @param idxAccess Index of the access rights of the segment in the VMCS.
6274 * @param pSelReg Pointer to the segment selector.
6275 *
6276 * @remarks No-long-jump zone!!!
6277 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6278 * macro as that takes care of whether to read from the VMCS cache or
6279 * not.
6280 */
6281DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6282 PCPUMSELREG pSelReg)
6283{
6284 NOREF(pVCpu);
6285
6286 uint32_t u32Val = 0;
6287 int rc = VMXReadVmcs32(idxSel, &u32Val);
6288 AssertRCReturn(rc, rc);
6289 pSelReg->Sel = (uint16_t)u32Val;
6290 pSelReg->ValidSel = (uint16_t)u32Val;
6291 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6292
6293 rc = VMXReadVmcs32(idxLimit, &u32Val);
6294 AssertRCReturn(rc, rc);
6295 pSelReg->u32Limit = u32Val;
6296
6297 uint64_t u64Val = 0;
6298 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6299 AssertRCReturn(rc, rc);
6300 pSelReg->u64Base = u64Val;
6301
6302 rc = VMXReadVmcs32(idxAccess, &u32Val);
6303 AssertRCReturn(rc, rc);
6304 pSelReg->Attr.u = u32Val;
6305
6306 /*
6307 * If VT-x marks the segment as unusable, most other bits remain undefined:
6308 * - For CS the L, D and G bits have meaning.
6309 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6310 * - For the remaining data segments no bits are defined.
6311 *
6312 * The present bit and the unusable bit has been observed to be set at the
6313 * same time (the selector was supposed to be invalid as we started executing
6314 * a V8086 interrupt in ring-0).
6315 *
6316 * What should be important for the rest of the VBox code, is that the P bit is
6317 * cleared. Some of the other VBox code recognizes the unusable bit, but
6318 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6319 * safe side here, we'll strip off P and other bits we don't care about. If
6320 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6321 *
6322 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6323 */
6324 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6325 {
6326 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6327
6328 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6329 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6330 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6331
6332 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6333#ifdef DEBUG_bird
6334 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6335 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6336 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6337#endif
6338 }
6339 return VINF_SUCCESS;
6340}
6341
6342
6343#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6344# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6345 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6346 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6347#else
6348# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6349 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6350 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6351#endif
6352
6353
6354/**
6355 * Saves the guest segment registers from the current VMCS into the guest-CPU
6356 * context.
6357 *
6358 * @returns VBox status code.
6359 * @param pVCpu Pointer to the VMCPU.
6360 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6361 * out-of-sync. Make sure to update the required fields
6362 * before using them.
6363 *
6364 * @remarks No-long-jump zone!!!
6365 */
6366static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6367{
6368 /* Guest segment registers. */
6369 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6370 {
6371 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6372 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6373 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6374 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6375 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6376 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6377 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6378
6379 /* Restore segment attributes for real-on-v86 mode hack. */
6380 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6381 {
6382 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6383 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6384 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6385 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6386 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6387 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6388 }
6389 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6390 }
6391
6392 return VINF_SUCCESS;
6393}
6394
6395
6396/**
6397 * Saves the guest descriptor table registers and task register from the current
6398 * VMCS into the guest-CPU context.
6399 *
6400 * @returns VBox status code.
6401 * @param pVCpu Pointer to the VMCPU.
6402 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6403 * out-of-sync. Make sure to update the required fields
6404 * before using them.
6405 *
6406 * @remarks No-long-jump zone!!!
6407 */
6408static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6409{
6410 int rc = VINF_SUCCESS;
6411
6412 /* Guest LDTR. */
6413 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6414 {
6415 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6416 AssertRCReturn(rc, rc);
6417 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6418 }
6419
6420 /* Guest GDTR. */
6421 uint64_t u64Val = 0;
6422 uint32_t u32Val = 0;
6423 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6424 {
6425 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6426 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6427 pMixedCtx->gdtr.pGdt = u64Val;
6428 pMixedCtx->gdtr.cbGdt = u32Val;
6429 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6430 }
6431
6432 /* Guest IDTR. */
6433 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6434 {
6435 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6436 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6437 pMixedCtx->idtr.pIdt = u64Val;
6438 pMixedCtx->idtr.cbIdt = u32Val;
6439 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6440 }
6441
6442 /* Guest TR. */
6443 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6444 {
6445 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6446 AssertRCReturn(rc, rc);
6447
6448 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6449 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6450 {
6451 rc = VMXLOCAL_READ_SEG(TR, tr);
6452 AssertRCReturn(rc, rc);
6453 }
6454 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6455 }
6456 return rc;
6457}
6458
6459#undef VMXLOCAL_READ_SEG
6460
6461
6462/**
6463 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6464 * context.
6465 *
6466 * @returns VBox status code.
6467 * @param pVCpu Pointer to the VMCPU.
6468 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6469 * out-of-sync. Make sure to update the required fields
6470 * before using them.
6471 *
6472 * @remarks No-long-jump zone!!!
6473 */
6474static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6475{
6476 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6477 {
6478 if (!pVCpu->hm.s.fUsingHyperDR7)
6479 {
6480 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6481 uint32_t u32Val;
6482 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6483 pMixedCtx->dr[7] = u32Val;
6484 }
6485
6486 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6487 }
6488 return VINF_SUCCESS;
6489}
6490
6491
6492/**
6493 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6494 *
6495 * @returns VBox status code.
6496 * @param pVCpu Pointer to the VMCPU.
6497 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6498 * out-of-sync. Make sure to update the required fields
6499 * before using them.
6500 *
6501 * @remarks No-long-jump zone!!!
6502 */
6503static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6504{
6505 NOREF(pMixedCtx);
6506
6507 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6508 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6509 return VINF_SUCCESS;
6510}
6511
6512
6513/**
6514 * Saves the entire guest state from the currently active VMCS into the
6515 * guest-CPU context.
6516 *
6517 * This essentially VMREADs all guest-data.
6518 *
6519 * @returns VBox status code.
6520 * @param pVCpu Pointer to the VMCPU.
6521 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6522 * out-of-sync. Make sure to update the required fields
6523 * before using them.
6524 */
6525static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6526{
6527 Assert(pVCpu);
6528 Assert(pMixedCtx);
6529
6530 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6531 return VINF_SUCCESS;
6532
6533 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6534 again on the ring-3 callback path, there is no real need to. */
6535 if (VMMRZCallRing3IsEnabled(pVCpu))
6536 VMMR0LogFlushDisable(pVCpu);
6537 else
6538 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6539 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6540
6541 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6542 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6543
6544 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6545 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6546
6547 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6548 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6549
6550 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6551 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6552
6553 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6554 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6555
6556 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6557 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6558
6559 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6560 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6561
6562 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6563 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6564
6565 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6566 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6567
6568 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6569 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6570
6571 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6572 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6573
6574 if (VMMRZCallRing3IsEnabled(pVCpu))
6575 VMMR0LogFlushEnable(pVCpu);
6576
6577 return VINF_SUCCESS;
6578}
6579
6580
6581/**
6582 * Saves basic guest registers needed for IEM instruction execution.
6583 *
6584 * @returns VBox status code (OR-able).
6585 * @param pVCpu Pointer to the cross context CPU data for the calling
6586 * EMT.
6587 * @param pMixedCtx Pointer to the CPU context of the guest.
6588 * @param fMemory Whether the instruction being executed operates on
6589 * memory or not. Only CR0 is synced up if clear.
6590 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6591 */
6592static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6593{
6594 /*
6595 * We assume all general purpose registers other than RSP are available.
6596 *
6597 * RIP is a must, as it will be incremented or otherwise changed.
6598 *
6599 * RFLAGS are always required to figure the CPL.
6600 *
6601 * RSP isn't always required, however it's a GPR, so frequently required.
6602 *
6603 * SS and CS are the only segment register needed if IEM doesn't do memory
6604 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6605 *
6606 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6607 * be required for memory accesses.
6608 *
6609 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6610 */
6611 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6612 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6613 if (fNeedRsp)
6614 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6615 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6616 if (!fMemory)
6617 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6618 else
6619 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6620 return rc;
6621}
6622
6623
6624/**
6625 * Ensures that we've got a complete basic guest-context.
6626 *
6627 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6628 * is for the interpreter.
6629 *
6630 * @returns VBox status code.
6631 * @param pVCpu Pointer to the VMCPU of the calling EMT.
6632 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6633 * needing to be synced in.
6634 * @thread EMT(pVCpu)
6635 */
6636VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6637{
6638 /* Note! Since this is only applicable to VT-x, the implementation is placed
6639 in the VT-x part of the sources instead of the generic stuff. */
6640 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6641 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6642 return VINF_SUCCESS;
6643}
6644
6645
6646/**
6647 * Check per-VM and per-VCPU force flag actions that require us to go back to
6648 * ring-3 for one reason or another.
6649 *
6650 * @returns VBox status code (information status code included).
6651 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6652 * ring-3.
6653 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6654 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6655 * interrupts)
6656 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6657 * all EMTs to be in ring-3.
6658 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6659 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6660 * to the EM loop.
6661 *
6662 * @param pVM Pointer to the VM.
6663 * @param pVCpu Pointer to the VMCPU.
6664 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6665 * out-of-sync. Make sure to update the required fields
6666 * before using them.
6667 */
6668static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6669{
6670 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6671
6672 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6673 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6674 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6675 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6676 {
6677 /* We need the control registers now, make sure the guest-CPU context is updated. */
6678 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6679 AssertRCReturn(rc3, rc3);
6680
6681 /* Pending HM CR3 sync. */
6682 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6683 {
6684 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6685 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6686 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6687 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6688 }
6689
6690 /* Pending HM PAE PDPEs. */
6691 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6692 {
6693 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6694 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6695 }
6696
6697 /* Pending PGM C3 sync. */
6698 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6699 {
6700 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6701 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6702 if (rc2 != VINF_SUCCESS)
6703 {
6704 AssertRC(rc2);
6705 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6706 return rc2;
6707 }
6708 }
6709
6710 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6711 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6712 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6713 {
6714 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6715 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6716 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6717 return rc2;
6718 }
6719
6720 /* Pending VM request packets, such as hardware interrupts. */
6721 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6722 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6723 {
6724 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6725 return VINF_EM_PENDING_REQUEST;
6726 }
6727
6728 /* Pending PGM pool flushes. */
6729 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6730 {
6731 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6732 return VINF_PGM_POOL_FLUSH_PENDING;
6733 }
6734
6735 /* Pending DMA requests. */
6736 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6737 {
6738 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6739 return VINF_EM_RAW_TO_R3;
6740 }
6741 }
6742
6743 return VINF_SUCCESS;
6744}
6745
6746
6747/**
6748 * Converts any TRPM trap into a pending HM event. This is typically used when
6749 * entering from ring-3 (not longjmp returns).
6750 *
6751 * @param pVCpu Pointer to the VMCPU.
6752 */
6753static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6754{
6755 Assert(TRPMHasTrap(pVCpu));
6756 Assert(!pVCpu->hm.s.Event.fPending);
6757
6758 uint8_t uVector;
6759 TRPMEVENT enmTrpmEvent;
6760 RTGCUINT uErrCode;
6761 RTGCUINTPTR GCPtrFaultAddress;
6762 uint8_t cbInstr;
6763
6764 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6765 AssertRC(rc);
6766
6767 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6768 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6769 if (enmTrpmEvent == TRPM_TRAP)
6770 {
6771 switch (uVector)
6772 {
6773 case X86_XCPT_NMI:
6774 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6775 break;
6776
6777 case X86_XCPT_BP:
6778 case X86_XCPT_OF:
6779 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6780 break;
6781
6782 case X86_XCPT_PF:
6783 case X86_XCPT_DF:
6784 case X86_XCPT_TS:
6785 case X86_XCPT_NP:
6786 case X86_XCPT_SS:
6787 case X86_XCPT_GP:
6788 case X86_XCPT_AC:
6789 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6790 /* no break! */
6791 default:
6792 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6793 break;
6794 }
6795 }
6796 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6797 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6798 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6799 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6800 else
6801 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6802
6803 rc = TRPMResetTrap(pVCpu);
6804 AssertRC(rc);
6805 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6806 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6807
6808 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6809 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6810}
6811
6812
6813/**
6814 * Converts the pending HM event into a TRPM trap.
6815 *
6816 * @param pvCpu Pointer to the VMCPU.
6817 */
6818static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6819{
6820 Assert(pVCpu->hm.s.Event.fPending);
6821
6822 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6823 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6824 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6825 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6826
6827 /* If a trap was already pending, we did something wrong! */
6828 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6829
6830 TRPMEVENT enmTrapType;
6831 switch (uVectorType)
6832 {
6833 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6834 enmTrapType = TRPM_HARDWARE_INT;
6835 break;
6836
6837 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6838 enmTrapType = TRPM_SOFTWARE_INT;
6839 break;
6840
6841 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6842 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6843 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6844 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6845 enmTrapType = TRPM_TRAP;
6846 break;
6847
6848 default:
6849 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6850 enmTrapType = TRPM_32BIT_HACK;
6851 break;
6852 }
6853
6854 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6855
6856 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6857 AssertRC(rc);
6858
6859 if (fErrorCodeValid)
6860 TRPMSetErrorCode(pVCpu, uErrorCode);
6861
6862 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6863 && uVector == X86_XCPT_PF)
6864 {
6865 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6866 }
6867 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6868 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6869 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6870 {
6871 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6872 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6873 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6874 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6875 }
6876
6877 /* Clear any pending events from the VMCS. */
6878 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6879 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6880
6881 /* We're now done converting the pending event. */
6882 pVCpu->hm.s.Event.fPending = false;
6883}
6884
6885
6886/**
6887 * Does the necessary state syncing before returning to ring-3 for any reason
6888 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6889 *
6890 * @returns VBox status code.
6891 * @param pVM Pointer to the VM.
6892 * @param pVCpu Pointer to the VMCPU.
6893 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6894 * be out-of-sync. Make sure to update the required
6895 * fields before using them.
6896 * @param fSaveGuestState Whether to save the guest state or not.
6897 *
6898 * @remarks No-long-jmp zone!!!
6899 */
6900static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6901{
6902 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6903 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6904
6905 RTCPUID idCpu = RTMpCpuId();
6906 Log4Func(("HostCpuId=%u\n", idCpu));
6907
6908 /*
6909 * !!! IMPORTANT !!!
6910 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6911 */
6912
6913 /* Save the guest state if necessary. */
6914 if ( fSaveGuestState
6915 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6916 {
6917 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6918 AssertRCReturn(rc, rc);
6919 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6920 }
6921
6922 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6923 if (CPUMIsGuestFPUStateActive(pVCpu))
6924 {
6925 /* We shouldn't reload CR0 without saving it first. */
6926 if (!fSaveGuestState)
6927 {
6928 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6929 AssertRCReturn(rc, rc);
6930 }
6931 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6932 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6933 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6934 }
6935
6936 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6937#ifdef VBOX_STRICT
6938 if (CPUMIsHyperDebugStateActive(pVCpu))
6939 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6940#endif
6941 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6942 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6943 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6944 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6945
6946#if HC_ARCH_BITS == 64
6947 /* Restore host-state bits that VT-x only restores partially. */
6948 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6949 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6950 {
6951 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6952 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6953 }
6954 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6955#endif
6956
6957#if HC_ARCH_BITS == 64
6958 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6959 if ( pVM->hm.s.fAllow64BitGuests
6960 && pVCpu->hm.s.vmx.fLazyMsrs)
6961 {
6962 /* We shouldn't reload the guest MSRs without saving it first. */
6963 if (!fSaveGuestState)
6964 {
6965 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6966 AssertRCReturn(rc, rc);
6967 }
6968 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6969 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6970 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6971 }
6972#endif
6973
6974 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6975 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6976
6977 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6978 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6979 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6980 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6981 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6982 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6983 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6984 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6985
6986 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6987
6988 /** @todo This partially defeats the purpose of having preemption hooks.
6989 * The problem is, deregistering the hooks should be moved to a place that
6990 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6991 * context.
6992 */
6993 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6994 {
6995 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6996 AssertRCReturn(rc, rc);
6997
6998 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6999 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7000 }
7001 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7002 NOREF(idCpu);
7003
7004 return VINF_SUCCESS;
7005}
7006
7007
7008/**
7009 * Leaves the VT-x session.
7010 *
7011 * @returns VBox status code.
7012 * @param pVM Pointer to the VM.
7013 * @param pVCpu Pointer to the VMCPU.
7014 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7015 * out-of-sync. Make sure to update the required fields
7016 * before using them.
7017 *
7018 * @remarks No-long-jmp zone!!!
7019 */
7020DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7021{
7022 HM_DISABLE_PREEMPT();
7023 HMVMX_ASSERT_CPU_SAFE();
7024 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7025 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7026
7027 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7028 and done this from the VMXR0ThreadCtxCallback(). */
7029 if (!pVCpu->hm.s.fLeaveDone)
7030 {
7031 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7032 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7033 pVCpu->hm.s.fLeaveDone = true;
7034 }
7035 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7036
7037 /*
7038 * !!! IMPORTANT !!!
7039 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7040 */
7041
7042 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7043 /** @todo Deregistering here means we need to VMCLEAR always
7044 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7045 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7046 VMMR0ThreadCtxHookDisable(pVCpu);
7047
7048 /* Leave HM context. This takes care of local init (term). */
7049 int rc = HMR0LeaveCpu(pVCpu);
7050
7051 HM_RESTORE_PREEMPT();
7052 return rc;
7053}
7054
7055
7056/**
7057 * Does the necessary state syncing before doing a longjmp to ring-3.
7058 *
7059 * @returns VBox status code.
7060 * @param pVM Pointer to the VM.
7061 * @param pVCpu Pointer to the VMCPU.
7062 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7063 * out-of-sync. Make sure to update the required fields
7064 * before using them.
7065 *
7066 * @remarks No-long-jmp zone!!!
7067 */
7068DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7069{
7070 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7071}
7072
7073
7074/**
7075 * Take necessary actions before going back to ring-3.
7076 *
7077 * An action requires us to go back to ring-3. This function does the necessary
7078 * steps before we can safely return to ring-3. This is not the same as longjmps
7079 * to ring-3, this is voluntary and prepares the guest so it may continue
7080 * executing outside HM (recompiler/IEM).
7081 *
7082 * @returns VBox status code.
7083 * @param pVM Pointer to the VM.
7084 * @param pVCpu Pointer to the VMCPU.
7085 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7086 * out-of-sync. Make sure to update the required fields
7087 * before using them.
7088 * @param rcExit The reason for exiting to ring-3. Can be
7089 * VINF_VMM_UNKNOWN_RING3_CALL.
7090 */
7091static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7092{
7093 Assert(pVM);
7094 Assert(pVCpu);
7095 Assert(pMixedCtx);
7096 HMVMX_ASSERT_PREEMPT_SAFE();
7097
7098 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7099 {
7100 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7101 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7102 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7103 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7104 }
7105
7106 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7107 VMMRZCallRing3Disable(pVCpu);
7108 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7109
7110 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7111 if (pVCpu->hm.s.Event.fPending)
7112 {
7113 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7114 Assert(!pVCpu->hm.s.Event.fPending);
7115 }
7116
7117 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7118 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7119
7120 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7121 and if we're injecting an event we should have a TRPM trap pending. */
7122 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", rcExit));
7123 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", rcExit));
7124
7125 /* Save guest state and restore host state bits. */
7126 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7127 AssertRCReturn(rc, rc);
7128 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7129 /* Thread-context hooks are unregistered at this point!!! */
7130
7131 /* Sync recompiler state. */
7132 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7133 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7134 | CPUM_CHANGED_LDTR
7135 | CPUM_CHANGED_GDTR
7136 | CPUM_CHANGED_IDTR
7137 | CPUM_CHANGED_TR
7138 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7139 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7140 if ( pVM->hm.s.fNestedPaging
7141 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7142 {
7143 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7144 }
7145
7146 Assert(!pVCpu->hm.s.fClearTrapFlag);
7147
7148 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7149 if (rcExit != VINF_EM_RAW_INTERRUPT)
7150 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7151
7152 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7153
7154 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7155 VMMRZCallRing3RemoveNotification(pVCpu);
7156 VMMRZCallRing3Enable(pVCpu);
7157
7158 return rc;
7159}
7160
7161
7162/**
7163 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7164 * longjump to ring-3 and possibly get preempted.
7165 *
7166 * @returns VBox status code.
7167 * @param pVCpu Pointer to the VMCPU.
7168 * @param enmOperation The operation causing the ring-3 longjump.
7169 * @param pvUser Opaque pointer to the guest-CPU context. The data
7170 * may be out-of-sync. Make sure to update the required
7171 * fields before using them.
7172 */
7173static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7174{
7175 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7176 {
7177 /*
7178 * !!! IMPORTANT !!!
7179 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7180 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7181 */
7182 VMMRZCallRing3RemoveNotification(pVCpu);
7183 VMMRZCallRing3Disable(pVCpu);
7184 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7185 RTThreadPreemptDisable(&PreemptState);
7186
7187 PVM pVM = pVCpu->CTX_SUFF(pVM);
7188 if (CPUMIsGuestFPUStateActive(pVCpu))
7189 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7190
7191 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7192
7193#if HC_ARCH_BITS == 64
7194 /* Restore host-state bits that VT-x only restores partially. */
7195 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7196 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7197 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7198 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7199
7200 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7201 if ( pVM->hm.s.fAllow64BitGuests
7202 && pVCpu->hm.s.vmx.fLazyMsrs)
7203 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7204#endif
7205 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7206 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7207 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7208 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7209 {
7210 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7211 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7212 }
7213
7214 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7215 VMMR0ThreadCtxHookDisable(pVCpu);
7216 HMR0LeaveCpu(pVCpu);
7217 RTThreadPreemptRestore(&PreemptState);
7218 return VINF_SUCCESS;
7219 }
7220
7221 Assert(pVCpu);
7222 Assert(pvUser);
7223 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7224 HMVMX_ASSERT_PREEMPT_SAFE();
7225
7226 VMMRZCallRing3Disable(pVCpu);
7227 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7228
7229 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7230 enmOperation));
7231
7232 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7233 AssertRCReturn(rc, rc);
7234
7235 VMMRZCallRing3Enable(pVCpu);
7236 return VINF_SUCCESS;
7237}
7238
7239
7240/**
7241 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7242 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7243 *
7244 * @param pVCpu Pointer to the VMCPU.
7245 */
7246DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7247{
7248 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7249 {
7250 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7251 {
7252 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7253 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7254 AssertRC(rc);
7255 Log4(("Setup interrupt-window exiting\n"));
7256 }
7257 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7258}
7259
7260
7261/**
7262 * Clears the interrupt-window exiting control in the VMCS.
7263 *
7264 * @param pVCpu Pointer to the VMCPU.
7265 */
7266DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7267{
7268 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7269 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7270 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7271 AssertRC(rc);
7272 Log4(("Cleared interrupt-window exiting\n"));
7273}
7274
7275
7276/**
7277 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7278 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7279 *
7280 * @param pVCpu Pointer to the VMCPU.
7281 */
7282DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7283{
7284 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7285 {
7286 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7287 {
7288 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7289 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7290 AssertRC(rc);
7291 Log4(("Setup NMI-window exiting\n"));
7292 }
7293 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7294}
7295
7296
7297/**
7298 * Clears the NMI-window exiting control in the VMCS.
7299 *
7300 * @param pVCpu Pointer to the VMCPU.
7301 */
7302DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7303{
7304 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7305 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7306 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7307 AssertRC(rc);
7308 Log4(("Cleared NMI-window exiting\n"));
7309}
7310
7311
7312/**
7313 * Evaluates the event to be delivered to the guest and sets it as the pending
7314 * event.
7315 *
7316 * @param pVCpu Pointer to the VMCPU.
7317 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7318 * out-of-sync. Make sure to update the required fields
7319 * before using them.
7320 */
7321static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7322{
7323 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7324 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7325 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7326 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7327 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7328
7329 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7330 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7331 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7332 Assert(!TRPMHasTrap(pVCpu));
7333
7334 /*
7335 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7336 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7337 */
7338 /** @todo SMI. SMIs take priority over NMIs. */
7339 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7340 {
7341 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7342 if ( !pVCpu->hm.s.Event.fPending
7343 && !fBlockNmi
7344 && !fBlockSti
7345 && !fBlockMovSS)
7346 {
7347 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7348 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7349 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7350
7351 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7352 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7353 }
7354 else
7355 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7356 }
7357 /*
7358 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7359 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7360 */
7361 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7362 && !pVCpu->hm.s.fSingleInstruction)
7363 {
7364 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7365 AssertRC(rc);
7366 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7367 if ( !pVCpu->hm.s.Event.fPending
7368 && !fBlockInt
7369 && !fBlockSti
7370 && !fBlockMovSS)
7371 {
7372 uint8_t u8Interrupt;
7373 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7374 if (RT_SUCCESS(rc))
7375 {
7376 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7377 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7378 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7379
7380 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7381 }
7382 else
7383 {
7384 /** @todo Does this actually happen? If not turn it into an assertion. */
7385 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7386 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7387 }
7388 }
7389 else
7390 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7391 }
7392}
7393
7394
7395/**
7396 * Sets a pending-debug exception to be delivered to the guest if the guest is
7397 * single-stepping.
7398 *
7399 * @param pVCpu Pointer to the VMCPU.
7400 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7401 * out-of-sync. Make sure to update the required fields
7402 * before using them.
7403 */
7404DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7405{
7406 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7407 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7408 {
7409 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7410 AssertRC(rc);
7411 }
7412}
7413
7414
7415/**
7416 * Injects any pending events into the guest if the guest is in a state to
7417 * receive them.
7418 *
7419 * @returns VBox status code (informational status codes included).
7420 * @param pVCpu Pointer to the VMCPU.
7421 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7422 * out-of-sync. Make sure to update the required fields
7423 * before using them.
7424 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7425 * return VINF_EM_DBG_STEPPED if the event was
7426 * dispatched directly.
7427 */
7428static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7429{
7430 HMVMX_ASSERT_PREEMPT_SAFE();
7431 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7432
7433 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7434 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7435 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7436 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7437
7438 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7439 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7440 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7441 Assert(!TRPMHasTrap(pVCpu));
7442
7443 int rc = VINF_SUCCESS;
7444 if (pVCpu->hm.s.Event.fPending)
7445 {
7446 /*
7447 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7448 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7449 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7450 *
7451 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7452 */
7453 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7454#ifdef VBOX_STRICT
7455 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7456 {
7457 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7458 Assert(!fBlockInt);
7459 Assert(!fBlockSti);
7460 Assert(!fBlockMovSS);
7461 }
7462 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7463 {
7464 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7465 Assert(!fBlockSti);
7466 Assert(!fBlockMovSS);
7467 Assert(!fBlockNmi);
7468 }
7469#endif
7470 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7471 (uint8_t)uIntType));
7472 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7473 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7474 AssertRCReturn(rc, rc);
7475
7476 /* Update the interruptibility-state as it could have been changed by
7477 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7478 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7479 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7480
7481 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7482 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7483 else
7484 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7485 }
7486
7487 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7488 if ( fBlockSti
7489 || fBlockMovSS)
7490 {
7491 if ( !pVCpu->hm.s.fSingleInstruction
7492 && !DBGFIsStepping(pVCpu))
7493 {
7494 /*
7495 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7496 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7497 * See Intel spec. 27.3.4 "Saving Non-Register State".
7498 */
7499 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7500 AssertRCReturn(rc2, rc2);
7501 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7502 }
7503 else if (pMixedCtx->eflags.Bits.u1TF)
7504 {
7505 /*
7506 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7507 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7508 */
7509 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7510 uIntrState = 0;
7511 }
7512 }
7513
7514 /*
7515 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7516 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7517 */
7518 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7519 AssertRC(rc2);
7520
7521 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7522 NOREF(fBlockMovSS); NOREF(fBlockSti);
7523 return rc;
7524}
7525
7526
7527/**
7528 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7529 *
7530 * @param pVCpu Pointer to the VMCPU.
7531 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7532 * out-of-sync. Make sure to update the required fields
7533 * before using them.
7534 */
7535DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7536{
7537 NOREF(pMixedCtx);
7538 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7539 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7540}
7541
7542
7543/**
7544 * Injects a double-fault (#DF) exception into the VM.
7545 *
7546 * @returns VBox status code (informational status code included).
7547 * @param pVCpu Pointer to the VMCPU.
7548 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7549 * out-of-sync. Make sure to update the required fields
7550 * before using them.
7551 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7552 * and should return VINF_EM_DBG_STEPPED if the event
7553 * is injected directly (register modified by us, not
7554 * by hardware on VM-entry).
7555 * @param puIntrState Pointer to the current guest interruptibility-state.
7556 * This interruptibility-state will be updated if
7557 * necessary. This cannot not be NULL.
7558 */
7559DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7560{
7561 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7562 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7563 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7564 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7565 fStepping, puIntrState);
7566}
7567
7568
7569/**
7570 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7571 *
7572 * @param pVCpu Pointer to the VMCPU.
7573 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7574 * out-of-sync. Make sure to update the required fields
7575 * before using them.
7576 */
7577DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7578{
7579 NOREF(pMixedCtx);
7580 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7581 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7582 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7583}
7584
7585
7586/**
7587 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7588 *
7589 * @param pVCpu Pointer to the VMCPU.
7590 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7591 * out-of-sync. Make sure to update the required fields
7592 * before using them.
7593 * @param cbInstr The value of RIP that is to be pushed on the guest
7594 * stack.
7595 */
7596DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7597{
7598 NOREF(pMixedCtx);
7599 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7600 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7601 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7602}
7603
7604
7605/**
7606 * Injects a general-protection (#GP) fault into the VM.
7607 *
7608 * @returns VBox status code (informational status code included).
7609 * @param pVCpu Pointer to the VMCPU.
7610 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7611 * out-of-sync. Make sure to update the required fields
7612 * before using them.
7613 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7614 * mode, i.e. in real-mode it's not valid).
7615 * @param u32ErrorCode The error code associated with the #GP.
7616 * @param fStepping Whether we're running in
7617 * hmR0VmxRunGuestCodeStep() and should return
7618 * VINF_EM_DBG_STEPPED if the event is injected
7619 * directly (register modified by us, not by
7620 * hardware on VM-entry).
7621 * @param puIntrState Pointer to the current guest interruptibility-state.
7622 * This interruptibility-state will be updated if
7623 * necessary. This cannot not be NULL.
7624 */
7625DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7626 bool fStepping, uint32_t *puIntrState)
7627{
7628 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7629 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7630 if (fErrorCodeValid)
7631 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7632 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7633 fStepping, puIntrState);
7634}
7635
7636
7637/**
7638 * Sets a general-protection (#GP) exception as pending-for-injection into the
7639 * VM.
7640 *
7641 * @param pVCpu Pointer to the VMCPU.
7642 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7643 * out-of-sync. Make sure to update the required fields
7644 * before using them.
7645 * @param u32ErrorCode The error code associated with the #GP.
7646 */
7647DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7648{
7649 NOREF(pMixedCtx);
7650 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7651 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7652 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7653 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7654}
7655
7656
7657/**
7658 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7659 *
7660 * @param pVCpu Pointer to the VMCPU.
7661 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7662 * out-of-sync. Make sure to update the required fields
7663 * before using them.
7664 * @param uVector The software interrupt vector number.
7665 * @param cbInstr The value of RIP that is to be pushed on the guest
7666 * stack.
7667 */
7668DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7669{
7670 NOREF(pMixedCtx);
7671 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7672 if ( uVector == X86_XCPT_BP
7673 || uVector == X86_XCPT_OF)
7674 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7675 else
7676 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7677 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7678}
7679
7680
7681/**
7682 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7683 * stack.
7684 *
7685 * @returns VBox status code (information status code included).
7686 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7687 * @param pVM Pointer to the VM.
7688 * @param pMixedCtx Pointer to the guest-CPU context.
7689 * @param uValue The value to push to the guest stack.
7690 */
7691DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7692{
7693 /*
7694 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7695 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7696 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7697 */
7698 if (pMixedCtx->sp == 1)
7699 return VINF_EM_RESET;
7700 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7701 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7702 AssertRCReturn(rc, rc);
7703 return rc;
7704}
7705
7706
7707/**
7708 * Injects an event into the guest upon VM-entry by updating the relevant fields
7709 * in the VM-entry area in the VMCS.
7710 *
7711 * @returns VBox status code (informational error codes included).
7712 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7713 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7714 *
7715 * @param pVCpu Pointer to the VMCPU.
7716 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7717 * be out-of-sync. Make sure to update the required
7718 * fields before using them.
7719 * @param u64IntInfo The VM-entry interruption-information field.
7720 * @param cbInstr The VM-entry instruction length in bytes (for
7721 * software interrupts, exceptions and privileged
7722 * software exceptions).
7723 * @param u32ErrCode The VM-entry exception error code.
7724 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7725 * @param puIntrState Pointer to the current guest interruptibility-state.
7726 * This interruptibility-state will be updated if
7727 * necessary. This cannot not be NULL.
7728 * @param fStepping Whether we're running in
7729 * hmR0VmxRunGuestCodeStep() and should return
7730 * VINF_EM_DBG_STEPPED if the event is injected
7731 * directly (register modified by us, not by
7732 * hardware on VM-entry).
7733 *
7734 * @remarks Requires CR0!
7735 * @remarks No-long-jump zone!!!
7736 */
7737static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7738 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7739{
7740 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7741 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7742 Assert(puIntrState);
7743 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7744
7745 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7746 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7747
7748#ifdef VBOX_STRICT
7749 /* Validate the error-code-valid bit for hardware exceptions. */
7750 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7751 {
7752 switch (uVector)
7753 {
7754 case X86_XCPT_PF:
7755 case X86_XCPT_DF:
7756 case X86_XCPT_TS:
7757 case X86_XCPT_NP:
7758 case X86_XCPT_SS:
7759 case X86_XCPT_GP:
7760 case X86_XCPT_AC:
7761 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7762 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7763 /* fallthru */
7764 default:
7765 break;
7766 }
7767 }
7768#endif
7769
7770 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7771 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7772 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7773
7774 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7775
7776 /* We require CR0 to check if the guest is in real-mode. */
7777 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7778 AssertRCReturn(rc, rc);
7779
7780 /*
7781 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7782 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7783 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7784 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7785 */
7786 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7787 {
7788 PVM pVM = pVCpu->CTX_SUFF(pVM);
7789 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7790 {
7791 Assert(PDMVmmDevHeapIsEnabled(pVM));
7792 Assert(pVM->hm.s.vmx.pRealModeTSS);
7793
7794 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7795 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7796 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7797 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7798 AssertRCReturn(rc, rc);
7799 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7800
7801 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7802 size_t const cbIdtEntry = sizeof(X86IDTR16);
7803 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7804 {
7805 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7806 if (uVector == X86_XCPT_DF)
7807 return VINF_EM_RESET;
7808
7809 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7810 if (uVector == X86_XCPT_GP)
7811 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7812
7813 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7814 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7815 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7816 fStepping, puIntrState);
7817 }
7818
7819 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7820 uint16_t uGuestIp = pMixedCtx->ip;
7821 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7822 {
7823 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7824 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7825 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7826 }
7827 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7828 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7829
7830 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7831 X86IDTR16 IdtEntry;
7832 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7833 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7834 AssertRCReturn(rc, rc);
7835
7836 /* Construct the stack frame for the interrupt/exception handler. */
7837 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7838 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7839 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7840 AssertRCReturn(rc, rc);
7841
7842 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7843 if (rc == VINF_SUCCESS)
7844 {
7845 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7846 pMixedCtx->rip = IdtEntry.offSel;
7847 pMixedCtx->cs.Sel = IdtEntry.uSel;
7848 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7849 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7850 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7851 && uVector == X86_XCPT_PF)
7852 pMixedCtx->cr2 = GCPtrFaultAddress;
7853
7854 /* If any other guest-state bits are changed here, make sure to update
7855 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7856 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7857 | HM_CHANGED_GUEST_RIP
7858 | HM_CHANGED_GUEST_RFLAGS
7859 | HM_CHANGED_GUEST_RSP);
7860
7861 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7862 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7863 {
7864 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7865 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7866 Log4(("Clearing inhibition due to STI.\n"));
7867 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7868 }
7869 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7870 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7871
7872 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7873 it, if we are returning to ring-3 before executing guest code. */
7874 pVCpu->hm.s.Event.fPending = false;
7875
7876 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7877 if (fStepping)
7878 rc = VINF_EM_DBG_STEPPED;
7879 }
7880 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7881 return rc;
7882 }
7883
7884 /*
7885 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7886 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7887 */
7888 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7889 }
7890
7891 /* Validate. */
7892 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7893 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7894 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7895
7896 /* Inject. */
7897 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7898 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7899 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7900 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7901
7902 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7903 && uVector == X86_XCPT_PF)
7904 pMixedCtx->cr2 = GCPtrFaultAddress;
7905
7906 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7907 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7908
7909 AssertRCReturn(rc, rc);
7910 return rc;
7911}
7912
7913
7914/**
7915 * Clears the interrupt-window exiting control in the VMCS and if necessary
7916 * clears the current event in the VMCS as well.
7917 *
7918 * @returns VBox status code.
7919 * @param pVCpu Pointer to the VMCPU.
7920 *
7921 * @remarks Use this function only to clear events that have not yet been
7922 * delivered to the guest but are injected in the VMCS!
7923 * @remarks No-long-jump zone!!!
7924 */
7925static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7926{
7927 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7928
7929 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7930 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7931
7932 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7933 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7934}
7935
7936
7937/**
7938 * Enters the VT-x session.
7939 *
7940 * @returns VBox status code.
7941 * @param pVM Pointer to the VM.
7942 * @param pVCpu Pointer to the VMCPU.
7943 * @param pCpu Pointer to the CPU info struct.
7944 */
7945VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7946{
7947 AssertPtr(pVM);
7948 AssertPtr(pVCpu);
7949 Assert(pVM->hm.s.vmx.fSupported);
7950 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7951 NOREF(pCpu); NOREF(pVM);
7952
7953 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7954 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7955
7956#ifdef VBOX_STRICT
7957 /* Make sure we're in VMX root mode. */
7958 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7959 if (!(u32HostCR4 & X86_CR4_VMXE))
7960 {
7961 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7962 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7963 }
7964#endif
7965
7966 /*
7967 * Load the VCPU's VMCS as the current (and active) one.
7968 */
7969 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7970 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7971 if (RT_FAILURE(rc))
7972 return rc;
7973
7974 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7975 pVCpu->hm.s.fLeaveDone = false;
7976 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7977
7978 return VINF_SUCCESS;
7979}
7980
7981
7982/**
7983 * The thread-context callback (only on platforms which support it).
7984 *
7985 * @param enmEvent The thread-context event.
7986 * @param pVCpu Pointer to the VMCPU.
7987 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7988 * @thread EMT(pVCpu)
7989 */
7990VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7991{
7992 NOREF(fGlobalInit);
7993
7994 switch (enmEvent)
7995 {
7996 case RTTHREADCTXEVENT_OUT:
7997 {
7998 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7999 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8000 VMCPU_ASSERT_EMT(pVCpu);
8001
8002 PVM pVM = pVCpu->CTX_SUFF(pVM);
8003 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8004
8005 /* No longjmps (logger flushes, locks) in this fragile context. */
8006 VMMRZCallRing3Disable(pVCpu);
8007 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8008
8009 /*
8010 * Restore host-state (FPU, debug etc.)
8011 */
8012 if (!pVCpu->hm.s.fLeaveDone)
8013 {
8014 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8015 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8016 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8017 pVCpu->hm.s.fLeaveDone = true;
8018 }
8019
8020 /* Leave HM context, takes care of local init (term). */
8021 int rc = HMR0LeaveCpu(pVCpu);
8022 AssertRC(rc); NOREF(rc);
8023
8024 /* Restore longjmp state. */
8025 VMMRZCallRing3Enable(pVCpu);
8026 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8027 break;
8028 }
8029
8030 case RTTHREADCTXEVENT_IN:
8031 {
8032 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8033 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8034 VMCPU_ASSERT_EMT(pVCpu);
8035
8036 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8037 VMMRZCallRing3Disable(pVCpu);
8038 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8039
8040 /* Initialize the bare minimum state required for HM. This takes care of
8041 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8042 int rc = HMR0EnterCpu(pVCpu);
8043 AssertRC(rc);
8044 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8045
8046 /* Load the active VMCS as the current one. */
8047 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8048 {
8049 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8050 AssertRC(rc); NOREF(rc);
8051 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8052 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8053 }
8054 pVCpu->hm.s.fLeaveDone = false;
8055
8056 /* Restore longjmp state. */
8057 VMMRZCallRing3Enable(pVCpu);
8058 break;
8059 }
8060
8061 default:
8062 break;
8063 }
8064}
8065
8066
8067/**
8068 * Saves the host state in the VMCS host-state.
8069 * Sets up the VM-exit MSR-load area.
8070 *
8071 * The CPU state will be loaded from these fields on every successful VM-exit.
8072 *
8073 * @returns VBox status code.
8074 * @param pVM Pointer to the VM.
8075 * @param pVCpu Pointer to the VMCPU.
8076 *
8077 * @remarks No-long-jump zone!!!
8078 */
8079static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8080{
8081 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8082
8083 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8084 return VINF_SUCCESS;
8085
8086 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8087 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8088
8089 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8090 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8091
8092 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8093 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8094
8095 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8096 return rc;
8097}
8098
8099
8100/**
8101 * Saves the host state in the VMCS host-state.
8102 *
8103 * @returns VBox status code.
8104 * @param pVM Pointer to the VM.
8105 * @param pVCpu Pointer to the VMCPU.
8106 *
8107 * @remarks No-long-jump zone!!!
8108 */
8109VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8110{
8111 AssertPtr(pVM);
8112 AssertPtr(pVCpu);
8113
8114 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8115
8116 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8117 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8118 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8119 return hmR0VmxSaveHostState(pVM, pVCpu);
8120}
8121
8122
8123/**
8124 * Loads the guest state into the VMCS guest-state area.
8125 *
8126 * The will typically be done before VM-entry when the guest-CPU state and the
8127 * VMCS state may potentially be out of sync.
8128 *
8129 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8130 * VM-entry controls.
8131 * Sets up the appropriate VMX non-root function to execute guest code based on
8132 * the guest CPU mode.
8133 *
8134 * @returns VBox status code.
8135 * @param pVM Pointer to the VM.
8136 * @param pVCpu Pointer to the VMCPU.
8137 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8138 * out-of-sync. Make sure to update the required fields
8139 * before using them.
8140 *
8141 * @remarks No-long-jump zone!!!
8142 */
8143static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8144{
8145 AssertPtr(pVM);
8146 AssertPtr(pVCpu);
8147 AssertPtr(pMixedCtx);
8148 HMVMX_ASSERT_PREEMPT_SAFE();
8149
8150 VMMRZCallRing3Disable(pVCpu);
8151 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8152
8153 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8154
8155 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8156
8157 /* Determine real-on-v86 mode. */
8158 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8159 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8160 && CPUMIsGuestInRealModeEx(pMixedCtx))
8161 {
8162 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8163 }
8164
8165 /*
8166 * Load the guest-state into the VMCS.
8167 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8168 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8169 */
8170 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8171 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8172
8173 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8174 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8175 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8176
8177 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8178 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8179 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8180
8181 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8182 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8183
8184 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8185 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8186
8187 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8188 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8189 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8190
8191 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8192 determine we don't have to swap EFER after all. */
8193 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8194 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8195
8196 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8197 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8198
8199 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8200 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8201
8202 /*
8203 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8204 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8205 */
8206 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8207 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8208
8209 /* Clear any unused and reserved bits. */
8210 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8211
8212 VMMRZCallRing3Enable(pVCpu);
8213
8214 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8215 return rc;
8216}
8217
8218
8219/**
8220 * Loads the state shared between the host and guest into the VMCS.
8221 *
8222 * @param pVM Pointer to the VM.
8223 * @param pVCpu Pointer to the VMCPU.
8224 * @param pCtx Pointer to the guest-CPU context.
8225 *
8226 * @remarks No-long-jump zone!!!
8227 */
8228static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8229{
8230 NOREF(pVM);
8231
8232 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8233 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8234
8235 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8236 {
8237 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8238 AssertRC(rc);
8239 }
8240
8241 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8242 {
8243 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8244 AssertRC(rc);
8245
8246 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8247 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8248 {
8249 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8250 AssertRC(rc);
8251 }
8252 }
8253
8254 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8255 {
8256#if HC_ARCH_BITS == 64
8257 if (pVM->hm.s.fAllow64BitGuests)
8258 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8259#endif
8260 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8261 }
8262
8263 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8264 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8265 {
8266 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8267 AssertRC(rc);
8268 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8269 }
8270
8271 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8272 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8273}
8274
8275
8276/**
8277 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8278 *
8279 * @param pVM Pointer to the VM.
8280 * @param pVCpu Pointer to the VMCPU.
8281 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8282 * out-of-sync. Make sure to update the required fields
8283 * before using them.
8284 */
8285DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8286{
8287 HMVMX_ASSERT_PREEMPT_SAFE();
8288
8289 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8290#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8291 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8292#endif
8293
8294 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8295 {
8296 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8297 AssertRC(rc);
8298 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8299 }
8300 else if (HMCPU_CF_VALUE(pVCpu))
8301 {
8302 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8303 AssertRC(rc);
8304 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8305 }
8306
8307 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8308 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8309 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8310 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8311}
8312
8313
8314/**
8315 * Does the preparations before executing guest code in VT-x.
8316 *
8317 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8318 * recompiler/IEM. We must be cautious what we do here regarding committing
8319 * guest-state information into the VMCS assuming we assuredly execute the
8320 * guest in VT-x mode.
8321 *
8322 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8323 * the common-state (TRPM/forceflags), we must undo those changes so that the
8324 * recompiler/IEM can (and should) use them when it resumes guest execution.
8325 * Otherwise such operations must be done when we can no longer exit to ring-3.
8326 *
8327 * @returns Strict VBox status code.
8328 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8329 * have been disabled.
8330 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8331 * double-fault into the guest.
8332 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8333 * dispatched directly.
8334 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8335 *
8336 * @param pVM Pointer to the VM.
8337 * @param pVCpu Pointer to the VMCPU.
8338 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8339 * out-of-sync. Make sure to update the required fields
8340 * before using them.
8341 * @param pVmxTransient Pointer to the VMX transient structure.
8342 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8343 * us ignore some of the reasons for returning to
8344 * ring-3, and return VINF_EM_DBG_STEPPED if event
8345 * dispatching took place.
8346 */
8347static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8348{
8349 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8350
8351#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8352 PGMRZDynMapFlushAutoSet(pVCpu);
8353#endif
8354
8355 /* Check force flag actions that might require us to go back to ring-3. */
8356 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8357 if (rc != VINF_SUCCESS)
8358 return rc;
8359
8360#ifndef IEM_VERIFICATION_MODE_FULL
8361 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8362 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8363 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8364 {
8365 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8366 RTGCPHYS GCPhysApicBase;
8367 GCPhysApicBase = pMixedCtx->msrApicBase;
8368 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8369
8370 /* Unalias any existing mapping. */
8371 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8372 AssertRCReturn(rc, rc);
8373
8374 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8375 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8376 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8377 AssertRCReturn(rc, rc);
8378
8379 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8380 }
8381#endif /* !IEM_VERIFICATION_MODE_FULL */
8382
8383 if (TRPMHasTrap(pVCpu))
8384 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8385 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8386
8387 /*
8388 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8389 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8390 */
8391 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8392 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8393 {
8394 Assert(rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
8395 return rc;
8396 }
8397
8398 /*
8399 * Load the guest state bits, we can handle longjmps/getting preempted here.
8400 *
8401 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8402 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8403 * Hence, this needs to be done -after- injection of events.
8404 */
8405 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8406
8407 /*
8408 * No longjmps to ring-3 from this point on!!!
8409 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8410 * This also disables flushing of the R0-logger instance (if any).
8411 */
8412 VMMRZCallRing3Disable(pVCpu);
8413
8414 /*
8415 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8416 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8417 *
8418 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8419 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8420 *
8421 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8422 * executing guest code.
8423 */
8424 pVmxTransient->fEFlags = ASMIntDisableFlags();
8425 if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8426 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8427 && ( !fStepping /* Optimized for the non-stepping case, of course. */
8428 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8429 {
8430 ASMSetFlags(pVmxTransient->fEFlags);
8431 VMMRZCallRing3Enable(pVCpu);
8432 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8433 return VINF_EM_RAW_TO_R3;
8434 }
8435
8436 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8437 {
8438 ASMSetFlags(pVmxTransient->fEFlags);
8439 VMMRZCallRing3Enable(pVCpu);
8440 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8441 return VINF_EM_RAW_INTERRUPT;
8442 }
8443
8444 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8445 pVCpu->hm.s.Event.fPending = false;
8446
8447 return VINF_SUCCESS;
8448}
8449
8450
8451/**
8452 * Prepares to run guest code in VT-x and we've committed to doing so. This
8453 * means there is no backing out to ring-3 or anywhere else at this
8454 * point.
8455 *
8456 * @param pVM Pointer to the VM.
8457 * @param pVCpu Pointer to the VMCPU.
8458 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8459 * out-of-sync. Make sure to update the required fields
8460 * before using them.
8461 * @param pVmxTransient Pointer to the VMX transient structure.
8462 *
8463 * @remarks Called with preemption disabled.
8464 * @remarks No-long-jump zone!!!
8465 */
8466static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8467{
8468 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8469 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8471
8472 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8473 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8474
8475#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8476 if (!CPUMIsGuestFPUStateActive(pVCpu))
8477 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8478 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8479#endif
8480
8481 if ( pVCpu->hm.s.fPreloadGuestFpu
8482 && !CPUMIsGuestFPUStateActive(pVCpu))
8483 {
8484 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8485 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8486 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8487 }
8488
8489 /*
8490 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8491 */
8492 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8493 && pVCpu->hm.s.vmx.cMsrs > 0)
8494 {
8495 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8496 }
8497
8498 /*
8499 * Load the host state bits as we may've been preempted (only happens when
8500 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8501 */
8502 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8503 * any effect to the host state needing to be saved? */
8504 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8505 {
8506 /* This ASSUMES that pfnStartVM has been set up already. */
8507 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8508 AssertRC(rc);
8509 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8510 }
8511 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8512
8513 /*
8514 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8515 */
8516 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8517 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8518 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8519
8520 /* Store status of the shared guest-host state at the time of VM-entry. */
8521#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8522 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8523 {
8524 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8525 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8526 }
8527 else
8528#endif
8529 {
8530 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8531 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8532 }
8533 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8534
8535 /*
8536 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8537 */
8538 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8539 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8540
8541 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8542 RTCPUID idCurrentCpu = pCpu->idCpu;
8543 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8544 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8545 {
8546 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8547 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8548 }
8549
8550 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8551 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8552 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8553 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8554
8555 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8556
8557 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8558 to start executing. */
8559
8560 /*
8561 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8562 */
8563 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8564 {
8565 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8566 {
8567 bool fMsrUpdated;
8568 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8569 AssertRC(rc2);
8570 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8571
8572 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8573 &fMsrUpdated);
8574 AssertRC(rc2);
8575 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8576
8577 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8578 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8579 }
8580 else
8581 {
8582 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8583 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8584 }
8585 }
8586
8587#ifdef VBOX_STRICT
8588 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8589 hmR0VmxCheckHostEferMsr(pVCpu);
8590 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8591#endif
8592#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8593 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8594 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8595 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8596#endif
8597}
8598
8599
8600/**
8601 * Performs some essential restoration of state after running guest code in
8602 * VT-x.
8603 *
8604 * @param pVM Pointer to the VM.
8605 * @param pVCpu Pointer to the VMCPU.
8606 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8607 * out-of-sync. Make sure to update the required fields
8608 * before using them.
8609 * @param pVmxTransient Pointer to the VMX transient structure.
8610 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8611 *
8612 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8613 *
8614 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8615 * unconditionally when it is safe to do so.
8616 */
8617static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8618{
8619 NOREF(pVM);
8620
8621 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8622
8623 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8624 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8625 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8626 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8627 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8628 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8629
8630 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8631 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8632
8633 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8634 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8635 Assert(!ASMIntAreEnabled());
8636 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8637
8638#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8639 if (CPUMIsGuestFPUStateActive(pVCpu))
8640 {
8641 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8642 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8643 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8644 }
8645#endif
8646
8647#if HC_ARCH_BITS == 64
8648 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8649#endif
8650 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8651#ifdef VBOX_STRICT
8652 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8653#endif
8654 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8655 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8656
8657 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8658 uint32_t uExitReason;
8659 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8660 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8661 AssertRC(rc);
8662 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8663 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8664
8665 /* Update the VM-exit history array. */
8666 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8667
8668 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8669 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8670 {
8671 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8672 pVmxTransient->fVMEntryFailed));
8673 return;
8674 }
8675
8676 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8677 {
8678 /** @todo We can optimize this by only syncing with our force-flags when
8679 * really needed and keeping the VMCS state as it is for most
8680 * VM-exits. */
8681 /* Update the guest interruptibility-state from the VMCS. */
8682 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8683
8684#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8685 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8686 AssertRC(rc);
8687#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8688 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8689 AssertRC(rc);
8690#endif
8691
8692 /*
8693 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8694 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8695 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8696 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8697 */
8698 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8699 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8700 {
8701 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8702 AssertRC(rc);
8703 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8704 }
8705 }
8706}
8707
8708
8709/**
8710 * Runs the guest code using VT-x the normal way.
8711 *
8712 * @returns VBox status code.
8713 * @param pVM Pointer to the VM.
8714 * @param pVCpu Pointer to the VMCPU.
8715 * @param pCtx Pointer to the guest-CPU context.
8716 *
8717 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8718 */
8719static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8720{
8721 VMXTRANSIENT VmxTransient;
8722 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8723 int rc = VERR_INTERNAL_ERROR_5;
8724 uint32_t cLoops = 0;
8725
8726 for (;; cLoops++)
8727 {
8728 Assert(!HMR0SuspendPending());
8729 HMVMX_ASSERT_CPU_SAFE();
8730
8731 /* Preparatory work for running guest code, this may force us to return
8732 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8733 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8734 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8735 if (rc != VINF_SUCCESS)
8736 break;
8737
8738 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8739 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8740 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8741
8742 /* Restore any residual host-state and save any bits shared between host
8743 and guest into the guest-CPU state. Re-enables interrupts! */
8744 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8745
8746 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8747 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8748 {
8749 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8750 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8751 return rc;
8752 }
8753
8754 /* Profile the VM-exit. */
8755 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8756 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8757 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8758 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8759 HMVMX_START_EXIT_DISPATCH_PROF();
8760
8761 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8762 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8763 {
8764 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8765 hmR0VmxSaveGuestState(pVCpu, pCtx);
8766 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8767 }
8768
8769 /* Handle the VM-exit. */
8770#ifdef HMVMX_USE_FUNCTION_TABLE
8771 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8772#else
8773 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8774#endif
8775 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8776 if (rc != VINF_SUCCESS)
8777 break;
8778 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8779 {
8780 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8781 rc = VINF_EM_RAW_INTERRUPT;
8782 break;
8783 }
8784 }
8785
8786 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8787 return rc;
8788}
8789
8790
8791/**
8792 * Single steps guest code using VT-x.
8793 *
8794 * @returns VBox status code.
8795 * @param pVM Pointer to the VM.
8796 * @param pVCpu Pointer to the VMCPU.
8797 * @param pCtx Pointer to the guest-CPU context.
8798 *
8799 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8800 */
8801static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8802{
8803 VMXTRANSIENT VmxTransient;
8804 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8805 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8806 uint32_t cLoops = 0;
8807 uint16_t uCsStart = pCtx->cs.Sel;
8808 uint64_t uRipStart = pCtx->rip;
8809
8810 for (;; cLoops++)
8811 {
8812 Assert(!HMR0SuspendPending());
8813 HMVMX_ASSERT_CPU_SAFE();
8814
8815 /* Preparatory work for running guest code, this may force us to return
8816 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8817 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8818 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8819 if (rcStrict != VINF_SUCCESS)
8820 break;
8821
8822 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8823 rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8824 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8825
8826 /* Restore any residual host-state and save any bits shared between host
8827 and guest into the guest-CPU state. Re-enables interrupts! */
8828 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8829
8830 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8831 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
8832 {
8833 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8834 hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
8835 return VBOXSTRICTRC_TODO(rcStrict);
8836 }
8837
8838 /* Profile the VM-exit. */
8839 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8840 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8841 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8842 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8843 HMVMX_START_EXIT_DISPATCH_PROF();
8844
8845 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8846 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8847 {
8848 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8849 hmR0VmxSaveGuestState(pVCpu, pCtx);
8850 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8851 }
8852
8853 /* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8854 rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8855 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8856 if (rcStrict != VINF_SUCCESS)
8857 break;
8858 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8859 {
8860 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8861 rcStrict = VINF_EM_RAW_INTERRUPT;
8862 break;
8863 }
8864
8865 /*
8866 * Did the RIP change, if so, consider it a single step.
8867 * Otherwise, make sure one of the TFs gets set.
8868 */
8869 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8870 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8871 AssertRCReturn(rc2, rc2);
8872 if ( pCtx->rip != uRipStart
8873 || pCtx->cs.Sel != uCsStart)
8874 {
8875 rcStrict = VINF_EM_DBG_STEPPED;
8876 break;
8877 }
8878 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8879 }
8880
8881 /*
8882 * Clear the X86_EFL_TF if necessary.
8883 */
8884 if (pVCpu->hm.s.fClearTrapFlag)
8885 {
8886 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8887 AssertRCReturn(rc2, rc2);
8888 pVCpu->hm.s.fClearTrapFlag = false;
8889 pCtx->eflags.Bits.u1TF = 0;
8890 }
8891 /** @todo there seems to be issues with the resume flag when the monitor trap
8892 * flag is pending without being used. Seen early in bios init when
8893 * accessing APIC page in protected mode. */
8894
8895 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8896 return VBOXSTRICTRC_TODO(rcStrict);
8897}
8898
8899
8900/**
8901 * Runs the guest code using VT-x.
8902 *
8903 * @returns VBox status code.
8904 * @param pVM Pointer to the VM.
8905 * @param pVCpu Pointer to the VMCPU.
8906 * @param pCtx Pointer to the guest-CPU context.
8907 */
8908VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8909{
8910 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8911 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8912 HMVMX_ASSERT_PREEMPT_SAFE();
8913
8914 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8915
8916 int rc;
8917 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8918 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8919 else
8920 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8921
8922 if (rc == VERR_EM_INTERPRETER)
8923 rc = VINF_EM_RAW_EMULATE_INSTR;
8924 else if (rc == VINF_EM_RESET)
8925 rc = VINF_EM_TRIPLE_FAULT;
8926
8927 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8928 if (RT_FAILURE(rc2))
8929 {
8930 pVCpu->hm.s.u32HMError = rc;
8931 rc = rc2;
8932 }
8933 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8934 return rc;
8935}
8936
8937
8938#ifndef HMVMX_USE_FUNCTION_TABLE
8939DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8940{
8941#ifdef DEBUG_ramshankar
8942# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8943# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8944#endif
8945 int rc;
8946 switch (rcReason)
8947 {
8948 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8949 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8950 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8951 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8952 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8953 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8954 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8955 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8956 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8957 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8958 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8959 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8960 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8961 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8962 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8963 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8964 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8965 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8966 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8967 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8968 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8969 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8970 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8971 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8972 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8973 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8974 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8975 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8976 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8977 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8978 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8979 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8980 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8981 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8982
8983 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8984 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8985 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8986 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8987 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8988 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8989 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8990 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8991 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8992
8993 case VMX_EXIT_VMCLEAR:
8994 case VMX_EXIT_VMLAUNCH:
8995 case VMX_EXIT_VMPTRLD:
8996 case VMX_EXIT_VMPTRST:
8997 case VMX_EXIT_VMREAD:
8998 case VMX_EXIT_VMRESUME:
8999 case VMX_EXIT_VMWRITE:
9000 case VMX_EXIT_VMXOFF:
9001 case VMX_EXIT_VMXON:
9002 case VMX_EXIT_INVEPT:
9003 case VMX_EXIT_INVVPID:
9004 case VMX_EXIT_VMFUNC:
9005 case VMX_EXIT_XSAVES:
9006 case VMX_EXIT_XRSTORS:
9007 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9008 break;
9009 case VMX_EXIT_RESERVED_60:
9010 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
9011 case VMX_EXIT_RESERVED_62:
9012 default:
9013 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9014 break;
9015 }
9016 return rc;
9017}
9018#endif /* !HMVMX_USE_FUNCTION_TABLE */
9019
9020
9021/**
9022 * Single-stepping VM-exit filtering.
9023 *
9024 * This is preprocessing the exits and deciding whether we've gotten far enough
9025 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9026 * performed.
9027 *
9028 * @returns Strict VBox status code.
9029 * @param pVCpu The virtual CPU of the calling EMT.
9030 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9031 * out-of-sync. Make sure to update the required
9032 * fields before using them.
9033 * @param pVmxTransient Pointer to the VMX-transient structure.
9034 * @param uExitReason The VM-exit reason.
9035 */
9036DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9037 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9038{
9039 switch (uExitReason)
9040 {
9041 case VMX_EXIT_XCPT_OR_NMI:
9042 {
9043 /* Check for host NMI. */
9044 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9045 AssertRCReturn(rc2, rc2);
9046 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9047 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9048 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9049 /* fall thru */
9050 }
9051
9052 case VMX_EXIT_EPT_MISCONFIG:
9053 case VMX_EXIT_TRIPLE_FAULT:
9054 case VMX_EXIT_APIC_ACCESS:
9055 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9056 case VMX_EXIT_TASK_SWITCH:
9057
9058 /* Instruction specific VM-exits: */
9059 case VMX_EXIT_IO_INSTR:
9060 case VMX_EXIT_CPUID:
9061 case VMX_EXIT_RDTSC:
9062 case VMX_EXIT_RDTSCP:
9063 case VMX_EXIT_MOV_CRX:
9064 case VMX_EXIT_MWAIT:
9065 case VMX_EXIT_MONITOR:
9066 case VMX_EXIT_RDMSR:
9067 case VMX_EXIT_WRMSR:
9068 case VMX_EXIT_MOV_DRX:
9069 case VMX_EXIT_HLT:
9070 case VMX_EXIT_INVD:
9071 case VMX_EXIT_INVLPG:
9072 case VMX_EXIT_RSM:
9073 case VMX_EXIT_PAUSE:
9074 case VMX_EXIT_XDTR_ACCESS:
9075 case VMX_EXIT_TR_ACCESS:
9076 case VMX_EXIT_WBINVD:
9077 case VMX_EXIT_XSETBV:
9078 case VMX_EXIT_RDRAND:
9079 case VMX_EXIT_INVPCID:
9080 case VMX_EXIT_GETSEC:
9081 case VMX_EXIT_RDPMC:
9082 case VMX_EXIT_VMCALL:
9083 case VMX_EXIT_VMCLEAR:
9084 case VMX_EXIT_VMLAUNCH:
9085 case VMX_EXIT_VMPTRLD:
9086 case VMX_EXIT_VMPTRST:
9087 case VMX_EXIT_VMREAD:
9088 case VMX_EXIT_VMRESUME:
9089 case VMX_EXIT_VMWRITE:
9090 case VMX_EXIT_VMXOFF:
9091 case VMX_EXIT_VMXON:
9092 case VMX_EXIT_INVEPT:
9093 case VMX_EXIT_INVVPID:
9094 case VMX_EXIT_VMFUNC:
9095 {
9096 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9097 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9098 AssertRCReturn(rc2, rc2);
9099 if ( pMixedCtx->rip != uRipStart
9100 || pMixedCtx->cs.Sel != uCsStart)
9101 return VINF_EM_DBG_STEPPED;
9102 break;
9103 }
9104 }
9105
9106 /*
9107 * Normal processing.
9108 */
9109#ifdef HMVMX_USE_FUNCTION_TABLE
9110 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9111#else
9112 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9113#endif
9114}
9115
9116
9117#ifdef DEBUG
9118/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9119# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9120 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9121
9122# define HMVMX_ASSERT_PREEMPT_CPUID() \
9123 do \
9124 { \
9125 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9126 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9127 } while (0)
9128
9129# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9130 do { \
9131 AssertPtr(pVCpu); \
9132 AssertPtr(pMixedCtx); \
9133 AssertPtr(pVmxTransient); \
9134 Assert(pVmxTransient->fVMEntryFailed == false); \
9135 Assert(ASMIntAreEnabled()); \
9136 HMVMX_ASSERT_PREEMPT_SAFE(); \
9137 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9138 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9139 HMVMX_ASSERT_PREEMPT_SAFE(); \
9140 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9141 HMVMX_ASSERT_PREEMPT_CPUID(); \
9142 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9143 } while (0)
9144
9145# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9146 do { \
9147 Log4Func(("\n")); \
9148 } while (0)
9149#else /* Release builds */
9150# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9151 do { \
9152 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9153 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9154 } while (0)
9155# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9156#endif
9157
9158
9159/**
9160 * Advances the guest RIP after reading it from the VMCS.
9161 *
9162 * @returns VBox status code.
9163 * @param pVCpu Pointer to the VMCPU.
9164 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9165 * out-of-sync. Make sure to update the required fields
9166 * before using them.
9167 * @param pVmxTransient Pointer to the VMX transient structure.
9168 *
9169 * @remarks No-long-jump zone!!!
9170 */
9171DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9172{
9173 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9174 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9175 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9176 AssertRCReturn(rc, rc);
9177
9178 pMixedCtx->rip += pVmxTransient->cbInstr;
9179 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9180
9181 /*
9182 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9183 * pending debug exception field as it takes care of priority of events.
9184 *
9185 * See Intel spec. 32.2.1 "Debug Exceptions".
9186 */
9187 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9188
9189 return rc;
9190}
9191
9192
9193/**
9194 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9195 * and update error record fields accordingly.
9196 *
9197 * @return VMX_IGS_* return codes.
9198 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9199 * wrong with the guest state.
9200 *
9201 * @param pVM Pointer to the VM.
9202 * @param pVCpu Pointer to the VMCPU.
9203 * @param pCtx Pointer to the guest-CPU state.
9204 *
9205 * @remarks This function assumes our cache of the VMCS controls
9206 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9207 */
9208static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9209{
9210#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9211#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9212 uError = (err); \
9213 break; \
9214 } else do { } while (0)
9215
9216 int rc;
9217 uint32_t uError = VMX_IGS_ERROR;
9218 uint32_t u32Val;
9219 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9220
9221 do
9222 {
9223 /*
9224 * CR0.
9225 */
9226 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9227 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9228 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9229 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9230 if (fUnrestrictedGuest)
9231 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9232
9233 uint32_t u32GuestCR0;
9234 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9235 AssertRCBreak(rc);
9236 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9237 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9238 if ( !fUnrestrictedGuest
9239 && (u32GuestCR0 & X86_CR0_PG)
9240 && !(u32GuestCR0 & X86_CR0_PE))
9241 {
9242 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9243 }
9244
9245 /*
9246 * CR4.
9247 */
9248 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9249 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9250
9251 uint32_t u32GuestCR4;
9252 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9253 AssertRCBreak(rc);
9254 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9255 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9256
9257 /*
9258 * IA32_DEBUGCTL MSR.
9259 */
9260 uint64_t u64Val;
9261 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9262 AssertRCBreak(rc);
9263 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9264 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9265 {
9266 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9267 }
9268 uint64_t u64DebugCtlMsr = u64Val;
9269
9270#ifdef VBOX_STRICT
9271 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9272 AssertRCBreak(rc);
9273 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9274#endif
9275 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9276
9277 /*
9278 * RIP and RFLAGS.
9279 */
9280 uint32_t u32Eflags;
9281#if HC_ARCH_BITS == 64
9282 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9283 AssertRCBreak(rc);
9284 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9285 if ( !fLongModeGuest
9286 || !pCtx->cs.Attr.n.u1Long)
9287 {
9288 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9289 }
9290 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9291 * must be identical if the "IA-32e mode guest" VM-entry
9292 * control is 1 and CS.L is 1. No check applies if the
9293 * CPU supports 64 linear-address bits. */
9294
9295 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9296 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9297 AssertRCBreak(rc);
9298 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9299 VMX_IGS_RFLAGS_RESERVED);
9300 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9301 u32Eflags = u64Val;
9302#else
9303 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9304 AssertRCBreak(rc);
9305 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9306 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9307#endif
9308
9309 if ( fLongModeGuest
9310 || ( fUnrestrictedGuest
9311 && !(u32GuestCR0 & X86_CR0_PE)))
9312 {
9313 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9314 }
9315
9316 uint32_t u32EntryInfo;
9317 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9318 AssertRCBreak(rc);
9319 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9320 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9321 {
9322 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9323 }
9324
9325 /*
9326 * 64-bit checks.
9327 */
9328#if HC_ARCH_BITS == 64
9329 if (fLongModeGuest)
9330 {
9331 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9332 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9333 }
9334
9335 if ( !fLongModeGuest
9336 && (u32GuestCR4 & X86_CR4_PCIDE))
9337 {
9338 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9339 }
9340
9341 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9342 * 51:32 beyond the processor's physical-address width are 0. */
9343
9344 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9345 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9346 {
9347 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9348 }
9349
9350 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9351 AssertRCBreak(rc);
9352 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9353
9354 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9355 AssertRCBreak(rc);
9356 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9357#endif
9358
9359 /*
9360 * PERF_GLOBAL MSR.
9361 */
9362 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9363 {
9364 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9365 AssertRCBreak(rc);
9366 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9367 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9368 }
9369
9370 /*
9371 * PAT MSR.
9372 */
9373 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9374 {
9375 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9376 AssertRCBreak(rc);
9377 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9378 for (unsigned i = 0; i < 8; i++)
9379 {
9380 uint8_t u8Val = (u64Val & 0xff);
9381 if ( u8Val != 0 /* UC */
9382 && u8Val != 1 /* WC */
9383 && u8Val != 4 /* WT */
9384 && u8Val != 5 /* WP */
9385 && u8Val != 6 /* WB */
9386 && u8Val != 7 /* UC- */)
9387 {
9388 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9389 }
9390 u64Val >>= 8;
9391 }
9392 }
9393
9394 /*
9395 * EFER MSR.
9396 */
9397 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9398 {
9399 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9400 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9401 AssertRCBreak(rc);
9402 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9403 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9404 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
9405 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9406 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9407 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9408 || !(u32GuestCR0 & X86_CR0_PG)
9409 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9410 VMX_IGS_EFER_LMA_LME_MISMATCH);
9411 }
9412
9413 /*
9414 * Segment registers.
9415 */
9416 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9417 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9418 if (!(u32Eflags & X86_EFL_VM))
9419 {
9420 /* CS */
9421 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9422 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9423 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9424 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9425 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9426 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9427 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9428 /* CS cannot be loaded with NULL in protected mode. */
9429 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9430 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9431 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9432 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9433 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9434 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9435 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9436 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9437 else
9438 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9439
9440 /* SS */
9441 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9442 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9443 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9444 if ( !(pCtx->cr0 & X86_CR0_PE)
9445 || pCtx->cs.Attr.n.u4Type == 3)
9446 {
9447 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9448 }
9449 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9450 {
9451 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9452 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9453 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9454 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9455 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9456 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9457 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9458 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9459 }
9460
9461 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9462 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9463 {
9464 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9465 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9466 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9467 || pCtx->ds.Attr.n.u4Type > 11
9468 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9469 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9470 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9471 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9472 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9473 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9474 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9475 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9476 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9477 }
9478 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9479 {
9480 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9481 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9482 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9483 || pCtx->es.Attr.n.u4Type > 11
9484 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9485 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9486 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9487 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9488 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9489 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9490 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9491 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9492 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9493 }
9494 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9495 {
9496 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9497 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9498 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9499 || pCtx->fs.Attr.n.u4Type > 11
9500 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9501 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9502 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9503 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9504 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9505 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9506 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9507 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9508 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9509 }
9510 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9511 {
9512 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9513 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9514 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9515 || pCtx->gs.Attr.n.u4Type > 11
9516 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9517 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9518 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9519 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9520 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9521 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9522 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9523 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9524 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9525 }
9526 /* 64-bit capable CPUs. */
9527#if HC_ARCH_BITS == 64
9528 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9529 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9530 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9531 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9532 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9533 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9534 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9535 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9536 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9537 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9538 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9539#endif
9540 }
9541 else
9542 {
9543 /* V86 mode checks. */
9544 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9545 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9546 {
9547 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9548 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9549 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9550 }
9551 else
9552 {
9553 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9554 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9555 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9556 }
9557
9558 /* CS */
9559 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9560 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9561 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9562 /* SS */
9563 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9564 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9565 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9566 /* DS */
9567 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9568 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9569 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9570 /* ES */
9571 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9572 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9573 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9574 /* FS */
9575 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9576 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9577 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9578 /* GS */
9579 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9580 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9581 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9582 /* 64-bit capable CPUs. */
9583#if HC_ARCH_BITS == 64
9584 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9585 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9586 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9587 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9588 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9589 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9590 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9591 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9592 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9593 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9594 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9595#endif
9596 }
9597
9598 /*
9599 * TR.
9600 */
9601 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9602 /* 64-bit capable CPUs. */
9603#if HC_ARCH_BITS == 64
9604 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9605#endif
9606 if (fLongModeGuest)
9607 {
9608 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9609 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9610 }
9611 else
9612 {
9613 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9614 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9615 VMX_IGS_TR_ATTR_TYPE_INVALID);
9616 }
9617 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9618 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9619 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9620 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9621 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9622 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9623 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9624 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9625
9626 /*
9627 * GDTR and IDTR.
9628 */
9629#if HC_ARCH_BITS == 64
9630 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9631 AssertRCBreak(rc);
9632 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9633
9634 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9635 AssertRCBreak(rc);
9636 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9637#endif
9638
9639 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9640 AssertRCBreak(rc);
9641 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9642
9643 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9644 AssertRCBreak(rc);
9645 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9646
9647 /*
9648 * Guest Non-Register State.
9649 */
9650 /* Activity State. */
9651 uint32_t u32ActivityState;
9652 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9653 AssertRCBreak(rc);
9654 HMVMX_CHECK_BREAK( !u32ActivityState
9655 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9656 VMX_IGS_ACTIVITY_STATE_INVALID);
9657 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9658 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9659 uint32_t u32IntrState;
9660 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9661 AssertRCBreak(rc);
9662 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9663 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9664 {
9665 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9666 }
9667
9668 /** @todo Activity state and injecting interrupts. Left as a todo since we
9669 * currently don't use activity states but ACTIVE. */
9670
9671 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9672 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9673
9674 /* Guest interruptibility-state. */
9675 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9676 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9677 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9678 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9679 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9680 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9681 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9682 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9683 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9684 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9685 {
9686 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9687 {
9688 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9689 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9690 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9691 }
9692 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9693 {
9694 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9695 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9696 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9697 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9698 }
9699 }
9700 /** @todo Assumes the processor is not in SMM. */
9701 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9702 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9703 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9704 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9705 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9706 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9707 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9708 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9709 {
9710 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9711 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9712 }
9713
9714 /* Pending debug exceptions. */
9715#if HC_ARCH_BITS == 64
9716 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9717 AssertRCBreak(rc);
9718 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9719 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9720 u32Val = u64Val; /* For pending debug exceptions checks below. */
9721#else
9722 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9723 AssertRCBreak(rc);
9724 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9725 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9726#endif
9727
9728 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9729 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9730 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9731 {
9732 if ( (u32Eflags & X86_EFL_TF)
9733 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9734 {
9735 /* Bit 14 is PendingDebug.BS. */
9736 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9737 }
9738 if ( !(u32Eflags & X86_EFL_TF)
9739 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9740 {
9741 /* Bit 14 is PendingDebug.BS. */
9742 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9743 }
9744 }
9745
9746 /* VMCS link pointer. */
9747 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9748 AssertRCBreak(rc);
9749 if (u64Val != UINT64_C(0xffffffffffffffff))
9750 {
9751 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9752 /** @todo Bits beyond the processor's physical-address width MBZ. */
9753 /** @todo 32-bit located in memory referenced by value of this field (as a
9754 * physical address) must contain the processor's VMCS revision ID. */
9755 /** @todo SMM checks. */
9756 }
9757
9758 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9759 * not using Nested Paging? */
9760 if ( pVM->hm.s.fNestedPaging
9761 && !fLongModeGuest
9762 && CPUMIsGuestInPAEModeEx(pCtx))
9763 {
9764 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9765 AssertRCBreak(rc);
9766 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9767
9768 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9769 AssertRCBreak(rc);
9770 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9771
9772 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9773 AssertRCBreak(rc);
9774 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9775
9776 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9777 AssertRCBreak(rc);
9778 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9779 }
9780
9781 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9782 if (uError == VMX_IGS_ERROR)
9783 uError = VMX_IGS_REASON_NOT_FOUND;
9784 } while (0);
9785
9786 pVCpu->hm.s.u32HMError = uError;
9787 return uError;
9788
9789#undef HMVMX_ERROR_BREAK
9790#undef HMVMX_CHECK_BREAK
9791}
9792
9793/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9794/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9795/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9796
9797/** @name VM-exit handlers.
9798 * @{
9799 */
9800
9801/**
9802 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9803 */
9804HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9805{
9806 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9807 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9808 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9809 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
9810 return VINF_SUCCESS;
9811 return VINF_EM_RAW_INTERRUPT;
9812}
9813
9814
9815/**
9816 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9817 */
9818HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9819{
9820 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9821 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9822
9823 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9824 AssertRCReturn(rc, rc);
9825
9826 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9827 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9828 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9829 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9830
9831 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9832 {
9833 /*
9834 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9835 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9836 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9837 *
9838 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9839 */
9840 VMXDispatchHostNmi();
9841 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9842 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9843 return VINF_SUCCESS;
9844 }
9845
9846 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9847 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9848 if (RT_UNLIKELY(rc != VINF_SUCCESS))
9849 {
9850 if (rc == VINF_HM_DOUBLE_FAULT)
9851 rc = VINF_SUCCESS;
9852 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9853 return rc;
9854 }
9855
9856 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9857 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9858 switch (uIntType)
9859 {
9860 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9861 Assert(uVector == X86_XCPT_DB);
9862 /* no break */
9863 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9864 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9865 /* no break */
9866 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9867 {
9868 switch (uVector)
9869 {
9870 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9871 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9872 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9873 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9874 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9875 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9876#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9877 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9878 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9879 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9880 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9881 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9882 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9883 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9884 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9885 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9886 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9887 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9888 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9889#endif
9890 default:
9891 {
9892 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9893 AssertRCReturn(rc, rc);
9894
9895 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9896 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9897 {
9898 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9899 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9900 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9901
9902 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9903 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9904 AssertRCReturn(rc, rc);
9905 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9906 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9907 0 /* GCPtrFaultAddress */);
9908 AssertRCReturn(rc, rc);
9909 }
9910 else
9911 {
9912 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9913 pVCpu->hm.s.u32HMError = uVector;
9914 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9915 }
9916 break;
9917 }
9918 }
9919 break;
9920 }
9921
9922 default:
9923 {
9924 pVCpu->hm.s.u32HMError = uExitIntInfo;
9925 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9926 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9927 break;
9928 }
9929 }
9930 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9931 return rc;
9932}
9933
9934
9935/**
9936 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9937 */
9938HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9939{
9940 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9941
9942 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9943 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9944
9945 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9946 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9947 return VINF_SUCCESS;
9948}
9949
9950
9951/**
9952 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9953 */
9954HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9955{
9956 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9957 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
9958 {
9959 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9960 HMVMX_RETURN_UNEXPECTED_EXIT();
9961 }
9962
9963 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
9964
9965 /*
9966 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
9967 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
9968 */
9969 uint32_t uIntrState = 0;
9970 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9971 AssertRCReturn(rc, rc);
9972
9973 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
9974 if ( fBlockSti
9975 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
9976 {
9977 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
9978 }
9979
9980 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
9981 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
9982
9983 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9984 return VINF_SUCCESS;
9985}
9986
9987
9988/**
9989 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9990 */
9991HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9992{
9993 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9994 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9995 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9996}
9997
9998
9999/**
10000 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10001 */
10002HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10003{
10004 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10005 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10006 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10007}
10008
10009
10010/**
10011 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10012 */
10013HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10014{
10015 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10016 PVM pVM = pVCpu->CTX_SUFF(pVM);
10017 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10018 if (RT_LIKELY(rc == VINF_SUCCESS))
10019 {
10020 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10021 Assert(pVmxTransient->cbInstr == 2);
10022 }
10023 else
10024 {
10025 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10026 rc = VERR_EM_INTERPRETER;
10027 }
10028 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10029 return rc;
10030}
10031
10032
10033/**
10034 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10035 */
10036HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10037{
10038 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10039 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10040 AssertRCReturn(rc, rc);
10041
10042 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10043 return VINF_EM_RAW_EMULATE_INSTR;
10044
10045 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10046 HMVMX_RETURN_UNEXPECTED_EXIT();
10047}
10048
10049
10050/**
10051 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10052 */
10053HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10054{
10055 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10056 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10057 AssertRCReturn(rc, rc);
10058
10059 PVM pVM = pVCpu->CTX_SUFF(pVM);
10060 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10061 if (RT_LIKELY(rc == VINF_SUCCESS))
10062 {
10063 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10064 Assert(pVmxTransient->cbInstr == 2);
10065 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10066 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10067 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10068 }
10069 else
10070 rc = VERR_EM_INTERPRETER;
10071 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10072 return rc;
10073}
10074
10075
10076/**
10077 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10078 */
10079HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10080{
10081 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10082 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10083 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10084 AssertRCReturn(rc, rc);
10085
10086 PVM pVM = pVCpu->CTX_SUFF(pVM);
10087 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10088 if (RT_LIKELY(rc == VINF_SUCCESS))
10089 {
10090 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10091 Assert(pVmxTransient->cbInstr == 3);
10092 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10093 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10094 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10095 }
10096 else
10097 {
10098 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10099 rc = VERR_EM_INTERPRETER;
10100 }
10101 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10102 return rc;
10103}
10104
10105
10106/**
10107 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10108 */
10109HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10110{
10111 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10112 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10113 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10114 AssertRCReturn(rc, rc);
10115
10116 PVM pVM = pVCpu->CTX_SUFF(pVM);
10117 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10118 if (RT_LIKELY(rc == VINF_SUCCESS))
10119 {
10120 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10121 Assert(pVmxTransient->cbInstr == 2);
10122 }
10123 else
10124 {
10125 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10126 rc = VERR_EM_INTERPRETER;
10127 }
10128 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10129 return rc;
10130}
10131
10132
10133/**
10134 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10135 */
10136HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10137{
10138 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10139 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10140
10141 if (pVCpu->hm.s.fHypercallsEnabled)
10142 {
10143#if 0
10144 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10145 AssertRCReturn(rc, rc);
10146#else
10147 /* Aggressive state sync. for now. */
10148 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10149 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
10150#endif
10151 AssertRCReturn(rc, rc);
10152
10153 rc = GIMHypercall(pVCpu, pMixedCtx);
10154 if (RT_SUCCESS(rc))
10155 {
10156 /* If the hypercall changes anything other than guest general-purpose registers,
10157 we would need to reload the guest changed bits here before VM-reentry. */
10158 hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10159 return VINF_SUCCESS;
10160 }
10161 }
10162
10163 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10164 return VINF_SUCCESS;
10165}
10166
10167
10168/**
10169 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10170 */
10171HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10172{
10173 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10174 PVM pVM = pVCpu->CTX_SUFF(pVM);
10175 Assert(!pVM->hm.s.fNestedPaging);
10176
10177 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10178 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10179 AssertRCReturn(rc, rc);
10180
10181 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10182 rc = VBOXSTRICTRC_VAL(rc2);
10183 if (RT_LIKELY(rc == VINF_SUCCESS))
10184 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10185 else
10186 {
10187 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10188 pVmxTransient->uExitQualification, rc));
10189 }
10190 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10191 return rc;
10192}
10193
10194
10195/**
10196 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10197 */
10198HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10199{
10200 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10201 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10202 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10203 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10204 AssertRCReturn(rc, rc);
10205
10206 PVM pVM = pVCpu->CTX_SUFF(pVM);
10207 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10208 if (RT_LIKELY(rc == VINF_SUCCESS))
10209 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10210 else
10211 {
10212 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10213 rc = VERR_EM_INTERPRETER;
10214 }
10215 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10216 return rc;
10217}
10218
10219
10220/**
10221 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10222 */
10223HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10224{
10225 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10226 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10227 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10228 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10229 AssertRCReturn(rc, rc);
10230
10231 PVM pVM = pVCpu->CTX_SUFF(pVM);
10232 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10233 rc = VBOXSTRICTRC_VAL(rc2);
10234 if (RT_LIKELY( rc == VINF_SUCCESS
10235 || rc == VINF_EM_HALT))
10236 {
10237 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10238 AssertRCReturn(rc3, rc3);
10239
10240 if ( rc == VINF_EM_HALT
10241 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10242 {
10243 rc = VINF_SUCCESS;
10244 }
10245 }
10246 else
10247 {
10248 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10249 rc = VERR_EM_INTERPRETER;
10250 }
10251 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10252 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10253 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10254 return rc;
10255}
10256
10257
10258/**
10259 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10260 */
10261HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10262{
10263 /*
10264 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10265 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10266 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10267 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10268 */
10269 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10270 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10271 HMVMX_RETURN_UNEXPECTED_EXIT();
10272}
10273
10274
10275/**
10276 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10277 */
10278HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10279{
10280 /*
10281 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10282 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10283 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10284 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10285 */
10286 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10287 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10288 HMVMX_RETURN_UNEXPECTED_EXIT();
10289}
10290
10291
10292/**
10293 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10294 */
10295HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10296{
10297 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10298 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10299 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10300 HMVMX_RETURN_UNEXPECTED_EXIT();
10301}
10302
10303
10304/**
10305 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10306 */
10307HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10308{
10309 /*
10310 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10311 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10312 * See Intel spec. 25.3 "Other Causes of VM-exits".
10313 */
10314 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10315 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10316 HMVMX_RETURN_UNEXPECTED_EXIT();
10317}
10318
10319
10320/**
10321 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10322 * VM-exit.
10323 */
10324HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10325{
10326 /*
10327 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10328 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10329 *
10330 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10331 * See Intel spec. "23.8 Restrictions on VMX operation".
10332 */
10333 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10334 return VINF_SUCCESS;
10335}
10336
10337
10338/**
10339 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10340 * VM-exit.
10341 */
10342HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10343{
10344 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10345 return VINF_EM_RESET;
10346}
10347
10348
10349/**
10350 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10351 */
10352HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10353{
10354 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10355 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10356 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10357 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10358 AssertRCReturn(rc, rc);
10359
10360 pMixedCtx->rip++;
10361 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10362 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10363 rc = VINF_SUCCESS;
10364 else
10365 rc = VINF_EM_HALT;
10366
10367 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10368 if (rc != VINF_SUCCESS)
10369 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
10370 return rc;
10371}
10372
10373
10374/**
10375 * VM-exit handler for instructions that result in a #UD exception delivered to
10376 * the guest.
10377 */
10378HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10379{
10380 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10381 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10382 return VINF_SUCCESS;
10383}
10384
10385
10386/**
10387 * VM-exit handler for expiry of the VMX preemption timer.
10388 */
10389HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10390{
10391 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10392
10393 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10394 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10395
10396 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10397 PVM pVM = pVCpu->CTX_SUFF(pVM);
10398 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10399 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10400 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10401}
10402
10403
10404/**
10405 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10406 */
10407HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10408{
10409 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10410
10411 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10412 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
10413 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10414 AssertRCReturn(rc, rc);
10415
10416 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
10417 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
10418
10419 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
10420
10421 return VBOXSTRICTRC_TODO(rcStrict);
10422}
10423
10424
10425/**
10426 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10427 */
10428HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10429{
10430 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10431
10432 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
10433 /** @todo implement EMInterpretInvpcid() */
10434 return VERR_EM_INTERPRETER;
10435}
10436
10437
10438/**
10439 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10440 * Error VM-exit.
10441 */
10442HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10443{
10444 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10445 AssertRCReturn(rc, rc);
10446
10447 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10448 AssertRCReturn(rc, rc);
10449
10450 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10451 NOREF(uInvalidReason);
10452
10453#ifdef VBOX_STRICT
10454 uint32_t uIntrState;
10455 RTHCUINTREG uHCReg;
10456 uint64_t u64Val;
10457 uint32_t u32Val;
10458
10459 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10460 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10461 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10462 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10463 AssertRCReturn(rc, rc);
10464
10465 Log4(("uInvalidReason %u\n", uInvalidReason));
10466 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10467 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10468 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10469 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10470
10471 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10472 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10473 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10474 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10475 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10476 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10477 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10478 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10479 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10480 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10481 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10482 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10483#else
10484 NOREF(pVmxTransient);
10485#endif
10486
10487 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10488 return VERR_VMX_INVALID_GUEST_STATE;
10489}
10490
10491
10492/**
10493 * VM-exit handler for VM-entry failure due to an MSR-load
10494 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10495 */
10496HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10497{
10498 NOREF(pVmxTransient);
10499 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10500 HMVMX_RETURN_UNEXPECTED_EXIT();
10501}
10502
10503
10504/**
10505 * VM-exit handler for VM-entry failure due to a machine-check event
10506 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10507 */
10508HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10509{
10510 NOREF(pVmxTransient);
10511 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10512 HMVMX_RETURN_UNEXPECTED_EXIT();
10513}
10514
10515
10516/**
10517 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10518 * theory.
10519 */
10520HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10521{
10522 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10523 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10524 return VERR_VMX_UNDEFINED_EXIT_CODE;
10525}
10526
10527
10528/**
10529 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10530 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10531 * Conditional VM-exit.
10532 */
10533HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10534{
10535 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10536
10537 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10538 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10539 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10540 return VERR_EM_INTERPRETER;
10541 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10542 HMVMX_RETURN_UNEXPECTED_EXIT();
10543}
10544
10545
10546/**
10547 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10548 */
10549HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10550{
10551 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10552
10553 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10554 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10555 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10556 return VERR_EM_INTERPRETER;
10557 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10558 HMVMX_RETURN_UNEXPECTED_EXIT();
10559}
10560
10561
10562/**
10563 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10564 */
10565HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10566{
10567 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10568
10569 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10570 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10571 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10572 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10573 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10574 {
10575 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10576 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10577 }
10578 AssertRCReturn(rc, rc);
10579 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10580
10581#ifdef VBOX_STRICT
10582 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10583 {
10584 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10585 && pMixedCtx->ecx != MSR_K6_EFER)
10586 {
10587 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10588 pMixedCtx->ecx));
10589 HMVMX_RETURN_UNEXPECTED_EXIT();
10590 }
10591# if HC_ARCH_BITS == 64
10592 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10593 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10594 {
10595 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10596 HMVMX_RETURN_UNEXPECTED_EXIT();
10597 }
10598# endif
10599 }
10600#endif
10601
10602 PVM pVM = pVCpu->CTX_SUFF(pVM);
10603 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10604 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10605 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10606 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10607 if (RT_LIKELY(rc == VINF_SUCCESS))
10608 {
10609 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10610 Assert(pVmxTransient->cbInstr == 2);
10611 }
10612 return rc;
10613}
10614
10615
10616/**
10617 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10618 */
10619HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10620{
10621 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10622 PVM pVM = pVCpu->CTX_SUFF(pVM);
10623 int rc = VINF_SUCCESS;
10624
10625 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10626 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10627 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10628 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10629 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10630 {
10631 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10632 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10633 }
10634 AssertRCReturn(rc, rc);
10635 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10636
10637 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10638 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10639 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10640
10641 if (RT_LIKELY(rc == VINF_SUCCESS))
10642 {
10643 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10644
10645 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10646 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10647 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10648 {
10649 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10650 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10651 EMInterpretWrmsr() changes it. */
10652 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10653 }
10654 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10655 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10656 else if (pMixedCtx->ecx == MSR_K6_EFER)
10657 {
10658 /*
10659 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10660 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10661 * the other bits as well, SCE and NXE. See @bugref{7368}.
10662 */
10663 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10664 }
10665
10666 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10667 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10668 {
10669 switch (pMixedCtx->ecx)
10670 {
10671 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10672 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10673 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10674 case MSR_K8_FS_BASE: /* no break */
10675 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10676 case MSR_K6_EFER: /* already handled above */ break;
10677 default:
10678 {
10679 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10680 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10681#if HC_ARCH_BITS == 64
10682 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10683 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10684#endif
10685 break;
10686 }
10687 }
10688 }
10689#ifdef VBOX_STRICT
10690 else
10691 {
10692 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10693 switch (pMixedCtx->ecx)
10694 {
10695 case MSR_IA32_SYSENTER_CS:
10696 case MSR_IA32_SYSENTER_EIP:
10697 case MSR_IA32_SYSENTER_ESP:
10698 case MSR_K8_FS_BASE:
10699 case MSR_K8_GS_BASE:
10700 {
10701 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10702 HMVMX_RETURN_UNEXPECTED_EXIT();
10703 }
10704
10705 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10706 default:
10707 {
10708 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10709 {
10710 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10711 if (pMixedCtx->ecx != MSR_K6_EFER)
10712 {
10713 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10714 pMixedCtx->ecx));
10715 HMVMX_RETURN_UNEXPECTED_EXIT();
10716 }
10717 }
10718
10719#if HC_ARCH_BITS == 64
10720 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10721 {
10722 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10723 HMVMX_RETURN_UNEXPECTED_EXIT();
10724 }
10725#endif
10726 break;
10727 }
10728 }
10729 }
10730#endif /* VBOX_STRICT */
10731 }
10732 return rc;
10733}
10734
10735
10736/**
10737 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10738 */
10739HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10740{
10741 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10742
10743 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10744 return VINF_EM_RAW_INTERRUPT;
10745}
10746
10747
10748/**
10749 * VM-exit handler for when the TPR value is lowered below the specified
10750 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10751 */
10752HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10753{
10754 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10755 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10756
10757 /*
10758 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10759 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10760 * resume guest execution.
10761 */
10762 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10763 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10764 return VINF_SUCCESS;
10765}
10766
10767
10768/**
10769 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10770 * VM-exit.
10771 *
10772 * @retval VINF_SUCCESS when guest execution can continue.
10773 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10774 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10775 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10776 * interpreter.
10777 */
10778HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10779{
10780 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10781 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10782 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10783 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10784 AssertRCReturn(rc, rc);
10785
10786 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10787 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10788 PVM pVM = pVCpu->CTX_SUFF(pVM);
10789 VBOXSTRICTRC rcStrict;
10790 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
10791 switch (uAccessType)
10792 {
10793 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10794 {
10795 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10796 AssertRCReturn(rc, rc);
10797
10798 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
10799 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10800 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10801 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
10802 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10803 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10804 {
10805 case 0: /* CR0 */
10806 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10807 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
10808 break;
10809 case 2: /* CR2 */
10810 /* Nothing to do here, CR2 it's not part of the VMCS. */
10811 break;
10812 case 3: /* CR3 */
10813 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10814 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10815 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
10816 break;
10817 case 4: /* CR4 */
10818 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10819 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
10820 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
10821 break;
10822 case 8: /* CR8 */
10823 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10824 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
10825 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10826 break;
10827 default:
10828 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10829 break;
10830 }
10831
10832 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10833 break;
10834 }
10835
10836 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10837 {
10838 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10839 AssertRCReturn(rc, rc);
10840
10841 Assert( !pVM->hm.s.fNestedPaging
10842 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10843 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10844
10845 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10846 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10847 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10848
10849 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
10850 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10851 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10852 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10853 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10854 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10855 VBOXSTRICTRC_VAL(rcStrict)));
10856 break;
10857 }
10858
10859 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10860 {
10861 AssertRCReturn(rc, rc);
10862 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
10863 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10864 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10865 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10866 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
10867 break;
10868 }
10869
10870 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10871 {
10872 AssertRCReturn(rc, rc);
10873 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
10874 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10875 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
10876 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10877 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10878 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
10879 break;
10880 }
10881
10882 default:
10883 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
10884 VERR_VMX_UNEXPECTED_EXCEPTION);
10885 }
10886
10887 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
10888 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10889 NOREF(pVM);
10890 return VBOXSTRICTRC_TODO(rcStrict);
10891}
10892
10893
10894/**
10895 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10896 * VM-exit.
10897 */
10898HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10899{
10900 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10901 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10902
10903 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10904 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10905 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10906 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10907 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10908 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10909 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10910 AssertRCReturn(rc2, rc2);
10911
10912 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10913 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10914 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10915 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10916 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10917 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10918 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10919 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10920
10921 /* I/O operation lookup arrays. */
10922 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10923 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10924
10925 VBOXSTRICTRC rcStrict;
10926 uint32_t const cbValue = s_aIOSizes[uIOWidth];
10927 uint32_t const cbInstr = pVmxTransient->cbInstr;
10928 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10929 PVM pVM = pVCpu->CTX_SUFF(pVM);
10930 if (fIOString)
10931 {
10932#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
10933 See @bugref{5752#c158}. Should work now. */
10934 /*
10935 * INS/OUTS - I/O String instruction.
10936 *
10937 * Use instruction-information if available, otherwise fall back on
10938 * interpreting the instruction.
10939 */
10940 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
10941 fIOWrite ? 'w' : 'r'));
10942 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
10943 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10944 {
10945 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10946 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10947 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10948 AssertRCReturn(rc2, rc2);
10949 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
10950 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10951 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10952 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10953 if (fIOWrite)
10954 {
10955 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10956 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10957 }
10958 else
10959 {
10960 /*
10961 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10962 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10963 * See Intel Instruction spec. for "INS".
10964 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10965 */
10966 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10967 }
10968 }
10969 else
10970 {
10971 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10972 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10973 AssertRCReturn(rc2, rc2);
10974 rcStrict = IEMExecOne(pVCpu);
10975 }
10976 /** @todo IEM needs to be setting these flags somehow. */
10977 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10978 fUpdateRipAlready = true;
10979#else
10980 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10981 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10982 if (RT_SUCCESS(rcStrict))
10983 {
10984 if (fIOWrite)
10985 {
10986 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10987 (DISCPUMODE)pDis->uAddrMode, cbValue);
10988 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10989 }
10990 else
10991 {
10992 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10993 (DISCPUMODE)pDis->uAddrMode, cbValue);
10994 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10995 }
10996 }
10997 else
10998 {
10999 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
11000 pMixedCtx->rip));
11001 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11002 }
11003#endif
11004 }
11005 else
11006 {
11007 /*
11008 * IN/OUT - I/O instruction.
11009 */
11010 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11011 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11012 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11013 if (fIOWrite)
11014 {
11015 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11016 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11017 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11018 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11019 }
11020 else
11021 {
11022 uint32_t u32Result = 0;
11023 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11024 if (IOM_SUCCESS(rcStrict))
11025 {
11026 /* Save result of I/O IN instr. in AL/AX/EAX. */
11027 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11028 }
11029 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11030 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11031 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11032 }
11033 }
11034
11035 if (IOM_SUCCESS(rcStrict))
11036 {
11037 if (!fUpdateRipAlready)
11038 {
11039 pMixedCtx->rip += cbInstr;
11040 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11041 }
11042
11043 /*
11044 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11045 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11046 */
11047 if (fIOString)
11048 {
11049 /** @todo Single-step for INS/OUTS with REP prefix? */
11050 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11051 }
11052 else if (fStepping)
11053 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11054
11055 /*
11056 * If any I/O breakpoints are armed, we need to check if one triggered
11057 * and take appropriate action.
11058 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11059 */
11060 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11061 AssertRCReturn(rc2, rc2);
11062
11063 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11064 * execution engines about whether hyper BPs and such are pending. */
11065 uint32_t const uDr7 = pMixedCtx->dr[7];
11066 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11067 && X86_DR7_ANY_RW_IO(uDr7)
11068 && (pMixedCtx->cr4 & X86_CR4_DE))
11069 || DBGFBpIsHwIoArmed(pVM)))
11070 {
11071 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11072
11073 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11074 VMMRZCallRing3Disable(pVCpu);
11075 HM_DISABLE_PREEMPT();
11076
11077 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11078
11079 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11080 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11081 {
11082 /* Raise #DB. */
11083 if (fIsGuestDbgActive)
11084 ASMSetDR6(pMixedCtx->dr[6]);
11085 if (pMixedCtx->dr[7] != uDr7)
11086 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11087
11088 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11089 }
11090 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11091 else if ( rcStrict2 != VINF_SUCCESS
11092 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11093 rcStrict = rcStrict2;
11094
11095 HM_RESTORE_PREEMPT();
11096 VMMRZCallRing3Enable(pVCpu);
11097 }
11098 }
11099
11100#ifdef DEBUG
11101 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11102 Assert(!fIOWrite);
11103 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11104 Assert(fIOWrite);
11105 else
11106 {
11107 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11108 * statuses, that the VMM device and some others may return. See
11109 * IOM_SUCCESS() for guidance. */
11110 AssertMsg( RT_FAILURE(rcStrict)
11111 || rcStrict == VINF_SUCCESS
11112 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11113 || rcStrict == VINF_EM_DBG_BREAKPOINT
11114 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11115 || rcStrict == VINF_EM_RAW_TO_R3
11116 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11117 }
11118#endif
11119
11120 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11121 return VBOXSTRICTRC_TODO(rcStrict);
11122}
11123
11124
11125/**
11126 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11127 * VM-exit.
11128 */
11129HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11130{
11131 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11132
11133 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11134 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11135 AssertRCReturn(rc, rc);
11136 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11137 {
11138 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11139 AssertRCReturn(rc, rc);
11140 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11141 {
11142 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11143
11144 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11145 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11146
11147 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11148 Assert(!pVCpu->hm.s.Event.fPending);
11149 pVCpu->hm.s.Event.fPending = true;
11150 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11151 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11152 AssertRCReturn(rc, rc);
11153 if (fErrorCodeValid)
11154 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11155 else
11156 pVCpu->hm.s.Event.u32ErrCode = 0;
11157 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11158 && uVector == X86_XCPT_PF)
11159 {
11160 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11161 }
11162
11163 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11164 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11165 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11166 }
11167 }
11168
11169 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11170 * emulation. */
11171 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11172 return VERR_EM_INTERPRETER;
11173}
11174
11175
11176/**
11177 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11178 */
11179HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11180{
11181 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11182 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11183 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11184 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11185 AssertRCReturn(rc, rc);
11186 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11187 return VINF_EM_DBG_STEPPED;
11188}
11189
11190
11191/**
11192 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11193 */
11194HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11195{
11196 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11197
11198 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11199 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11200 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11201 {
11202 if (rc == VINF_HM_DOUBLE_FAULT)
11203 rc = VINF_SUCCESS;
11204 return rc;
11205 }
11206
11207#if 0
11208 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11209 * just sync the whole thing. */
11210 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11211#else
11212 /* Aggressive state sync. for now. */
11213 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11214 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11215 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11216#endif
11217 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11218 AssertRCReturn(rc, rc);
11219
11220 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11221 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11222 switch (uAccessType)
11223 {
11224 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11225 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11226 {
11227 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11228 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11229 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11230
11231 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11232 GCPhys &= PAGE_BASE_GC_MASK;
11233 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11234 PVM pVM = pVCpu->CTX_SUFF(pVM);
11235 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11236 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11237
11238 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11239 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11240 CPUMCTX2CORE(pMixedCtx), GCPhys);
11241 rc = VBOXSTRICTRC_VAL(rc2);
11242 Log4(("ApicAccess rc=%d\n", rc));
11243 if ( rc == VINF_SUCCESS
11244 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11245 || rc == VERR_PAGE_NOT_PRESENT)
11246 {
11247 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11248 | HM_CHANGED_GUEST_RSP
11249 | HM_CHANGED_GUEST_RFLAGS
11250 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11251 rc = VINF_SUCCESS;
11252 }
11253 break;
11254 }
11255
11256 default:
11257 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11258 rc = VINF_EM_RAW_EMULATE_INSTR;
11259 break;
11260 }
11261
11262 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11263 if (rc != VINF_SUCCESS)
11264 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
11265 return rc;
11266}
11267
11268
11269/**
11270 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11271 * VM-exit.
11272 */
11273HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11274{
11275 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11276
11277 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11278 if (pVmxTransient->fWasGuestDebugStateActive)
11279 {
11280 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11281 HMVMX_RETURN_UNEXPECTED_EXIT();
11282 }
11283
11284 int rc = VERR_INTERNAL_ERROR_5;
11285 if ( !DBGFIsStepping(pVCpu)
11286 && !pVCpu->hm.s.fSingleInstruction
11287 && !pVmxTransient->fWasHyperDebugStateActive)
11288 {
11289 /* Don't intercept MOV DRx and #DB any more. */
11290 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11291 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11292 AssertRCReturn(rc, rc);
11293
11294 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11295 {
11296#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11297 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11298 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
11299#endif
11300 }
11301
11302 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11303 VMMRZCallRing3Disable(pVCpu);
11304 HM_DISABLE_PREEMPT();
11305
11306 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11307 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11308 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11309
11310 HM_RESTORE_PREEMPT();
11311 VMMRZCallRing3Enable(pVCpu);
11312
11313#ifdef VBOX_WITH_STATISTICS
11314 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11315 AssertRCReturn(rc, rc);
11316 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11317 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11318 else
11319 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11320#endif
11321 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11322 return VINF_SUCCESS;
11323 }
11324
11325 /*
11326 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11327 * Update the segment registers and DR7 from the CPU.
11328 */
11329 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11330 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11331 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11332 AssertRCReturn(rc, rc);
11333 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11334
11335 PVM pVM = pVCpu->CTX_SUFF(pVM);
11336 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11337 {
11338 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11339 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11340 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11341 if (RT_SUCCESS(rc))
11342 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11343 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11344 }
11345 else
11346 {
11347 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11348 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11349 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11350 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11351 }
11352
11353 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11354 if (RT_SUCCESS(rc))
11355 {
11356 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11357 AssertRCReturn(rc2, rc2);
11358 }
11359 return rc;
11360}
11361
11362
11363/**
11364 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11365 * Conditional VM-exit.
11366 */
11367HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11368{
11369 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11370 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11371
11372 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11373 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11374 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11375 {
11376 if (rc == VINF_HM_DOUBLE_FAULT)
11377 rc = VINF_SUCCESS;
11378 return rc;
11379 }
11380
11381 RTGCPHYS GCPhys = 0;
11382 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11383
11384#if 0
11385 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11386#else
11387 /* Aggressive state sync. for now. */
11388 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11389 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11390 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11391#endif
11392 AssertRCReturn(rc, rc);
11393
11394 /*
11395 * If we succeed, resume guest execution.
11396 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11397 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11398 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11399 * weird case. See @bugref{6043}.
11400 */
11401 PVM pVM = pVCpu->CTX_SUFF(pVM);
11402 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11403 rc = VBOXSTRICTRC_VAL(rc2);
11404 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11405 if ( rc == VINF_SUCCESS
11406 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11407 || rc == VERR_PAGE_NOT_PRESENT)
11408 {
11409 /* Successfully handled MMIO operation. */
11410 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11411 | HM_CHANGED_GUEST_RSP
11412 | HM_CHANGED_GUEST_RFLAGS
11413 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11414 rc = VINF_SUCCESS;
11415 }
11416 return rc;
11417}
11418
11419
11420/**
11421 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11422 * VM-exit.
11423 */
11424HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11425{
11426 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11427 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11428
11429 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11430 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11431 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11432 {
11433 if (rc == VINF_HM_DOUBLE_FAULT)
11434 rc = VINF_SUCCESS;
11435 return rc;
11436 }
11437
11438 RTGCPHYS GCPhys = 0;
11439 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11440 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11441#if 0
11442 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11443#else
11444 /* Aggressive state sync. for now. */
11445 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11446 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11447 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11448#endif
11449 AssertRCReturn(rc, rc);
11450
11451 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11452 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11453
11454 RTGCUINT uErrorCode = 0;
11455 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11456 uErrorCode |= X86_TRAP_PF_ID;
11457 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11458 uErrorCode |= X86_TRAP_PF_RW;
11459 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11460 uErrorCode |= X86_TRAP_PF_P;
11461
11462 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11463
11464 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11465 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11466
11467 /* Handle the pagefault trap for the nested shadow table. */
11468 PVM pVM = pVCpu->CTX_SUFF(pVM);
11469 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11470 TRPMResetTrap(pVCpu);
11471
11472 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11473 if ( rc == VINF_SUCCESS
11474 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11475 || rc == VERR_PAGE_NOT_PRESENT)
11476 {
11477 /* Successfully synced our nested page tables. */
11478 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11479 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11480 | HM_CHANGED_GUEST_RSP
11481 | HM_CHANGED_GUEST_RFLAGS);
11482 return VINF_SUCCESS;
11483 }
11484
11485 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11486 return rc;
11487}
11488
11489/** @} */
11490
11491/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11492/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11493/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11494
11495/** @name VM-exit exception handlers.
11496 * @{
11497 */
11498
11499/**
11500 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11501 */
11502static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11503{
11504 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11505 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11506
11507 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11508 AssertRCReturn(rc, rc);
11509
11510 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11511 {
11512 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11513 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11514
11515 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11516 * provides VM-exit instruction length. If this causes problem later,
11517 * disassemble the instruction like it's done on AMD-V. */
11518 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11519 AssertRCReturn(rc2, rc2);
11520 return rc;
11521 }
11522
11523 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11524 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11525 return rc;
11526}
11527
11528
11529/**
11530 * VM-exit exception handler for #BP (Breakpoint exception).
11531 */
11532static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11533{
11534 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11535 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11536
11537 /** @todo Try optimize this by not saving the entire guest state unless
11538 * really needed. */
11539 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11540 AssertRCReturn(rc, rc);
11541
11542 PVM pVM = pVCpu->CTX_SUFF(pVM);
11543 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11544 if (rc == VINF_EM_RAW_GUEST_TRAP)
11545 {
11546 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11547 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11548 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11549 AssertRCReturn(rc, rc);
11550
11551 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11552 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11553 }
11554
11555 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11556 return rc;
11557}
11558
11559
11560/**
11561 * VM-exit exception handler for #DB (Debug exception).
11562 */
11563static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11564{
11565 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11566 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11567 Log6(("XcptDB\n"));
11568
11569 /*
11570 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11571 * for processing.
11572 */
11573 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11574 AssertRCReturn(rc, rc);
11575
11576 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11577 uint64_t uDR6 = X86_DR6_INIT_VAL;
11578 uDR6 |= ( pVmxTransient->uExitQualification
11579 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11580
11581 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11582 if (rc == VINF_EM_RAW_GUEST_TRAP)
11583 {
11584 /*
11585 * The exception was for the guest. Update DR6, DR7.GD and
11586 * IA32_DEBUGCTL.LBR before forwarding it.
11587 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11588 */
11589 VMMRZCallRing3Disable(pVCpu);
11590 HM_DISABLE_PREEMPT();
11591
11592 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11593 pMixedCtx->dr[6] |= uDR6;
11594 if (CPUMIsGuestDebugStateActive(pVCpu))
11595 ASMSetDR6(pMixedCtx->dr[6]);
11596
11597 HM_RESTORE_PREEMPT();
11598 VMMRZCallRing3Enable(pVCpu);
11599
11600 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11601 AssertRCReturn(rc, rc);
11602
11603 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11604 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11605
11606 /* Paranoia. */
11607 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11608 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11609
11610 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11611 AssertRCReturn(rc, rc);
11612
11613 /*
11614 * Raise #DB in the guest.
11615 *
11616 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11617 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11618 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11619 *
11620 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11621 */
11622 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11623 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11624 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11625 AssertRCReturn(rc, rc);
11626 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11627 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11628 return VINF_SUCCESS;
11629 }
11630
11631 /*
11632 * Not a guest trap, must be a hypervisor related debug event then.
11633 * Update DR6 in case someone is interested in it.
11634 */
11635 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11636 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11637 CPUMSetHyperDR6(pVCpu, uDR6);
11638
11639 return rc;
11640}
11641
11642
11643/**
11644 * VM-exit exception handler for #NM (Device-not-available exception: floating
11645 * point exception).
11646 */
11647static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11648{
11649 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11650
11651 /* We require CR0 and EFER. EFER is always up-to-date. */
11652 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11653 AssertRCReturn(rc, rc);
11654
11655 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11656 VMMRZCallRing3Disable(pVCpu);
11657 HM_DISABLE_PREEMPT();
11658
11659 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11660 if (pVmxTransient->fWasGuestFPUStateActive)
11661 {
11662 rc = VINF_EM_RAW_GUEST_TRAP;
11663 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11664 }
11665 else
11666 {
11667#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11668 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11669#endif
11670 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11671 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11672 }
11673
11674 HM_RESTORE_PREEMPT();
11675 VMMRZCallRing3Enable(pVCpu);
11676
11677 if (rc == VINF_SUCCESS)
11678 {
11679 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11680 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11681 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11682 pVCpu->hm.s.fPreloadGuestFpu = true;
11683 }
11684 else
11685 {
11686 /* Forward #NM to the guest. */
11687 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11688 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11689 AssertRCReturn(rc, rc);
11690 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11691 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11692 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11693 }
11694
11695 return VINF_SUCCESS;
11696}
11697
11698
11699/**
11700 * VM-exit exception handler for #GP (General-protection exception).
11701 *
11702 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11703 */
11704static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11705{
11706 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11707 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11708
11709 int rc = VERR_INTERNAL_ERROR_5;
11710 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11711 {
11712#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11713 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11714 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11715 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11716 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11717 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11718 AssertRCReturn(rc, rc);
11719 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11720 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11721 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11722 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11723 return rc;
11724#else
11725 /* We don't intercept #GP. */
11726 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11727 NOREF(pVmxTransient);
11728 return VERR_VMX_UNEXPECTED_EXCEPTION;
11729#endif
11730 }
11731
11732 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11733 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11734
11735 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11736 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11737 AssertRCReturn(rc, rc);
11738
11739 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11740 uint32_t cbOp = 0;
11741 PVM pVM = pVCpu->CTX_SUFF(pVM);
11742 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11743 if (RT_SUCCESS(rc))
11744 {
11745 rc = VINF_SUCCESS;
11746 Assert(cbOp == pDis->cbInstr);
11747 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11748 switch (pDis->pCurInstr->uOpcode)
11749 {
11750 case OP_CLI:
11751 {
11752 pMixedCtx->eflags.Bits.u1IF = 0;
11753 pMixedCtx->eflags.Bits.u1RF = 0;
11754 pMixedCtx->rip += pDis->cbInstr;
11755 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11756 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11757 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11758 break;
11759 }
11760
11761 case OP_STI:
11762 {
11763 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11764 pMixedCtx->eflags.Bits.u1IF = 1;
11765 pMixedCtx->eflags.Bits.u1RF = 0;
11766 pMixedCtx->rip += pDis->cbInstr;
11767 if (!fOldIF)
11768 {
11769 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11770 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11771 }
11772 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11773 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11774 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11775 break;
11776 }
11777
11778 case OP_HLT:
11779 {
11780 rc = VINF_EM_HALT;
11781 pMixedCtx->rip += pDis->cbInstr;
11782 pMixedCtx->eflags.Bits.u1RF = 0;
11783 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11784 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11785 break;
11786 }
11787
11788 case OP_POPF:
11789 {
11790 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11791 uint32_t cbParm;
11792 uint32_t uMask;
11793 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11794 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11795 {
11796 cbParm = 4;
11797 uMask = 0xffffffff;
11798 }
11799 else
11800 {
11801 cbParm = 2;
11802 uMask = 0xffff;
11803 }
11804
11805 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11806 RTGCPTR GCPtrStack = 0;
11807 X86EFLAGS Eflags;
11808 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11809 &GCPtrStack);
11810 if (RT_SUCCESS(rc))
11811 {
11812 Assert(sizeof(Eflags.u32) >= cbParm);
11813 Eflags.u32 = 0;
11814 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
11815 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
11816 }
11817 if (RT_FAILURE(rc))
11818 {
11819 rc = VERR_EM_INTERPRETER;
11820 break;
11821 }
11822 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11823 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
11824 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11825 pMixedCtx->esp += cbParm;
11826 pMixedCtx->esp &= uMask;
11827 pMixedCtx->rip += pDis->cbInstr;
11828 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11829 | HM_CHANGED_GUEST_RSP
11830 | HM_CHANGED_GUEST_RFLAGS);
11831 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11832 if (fStepping)
11833 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11834
11835 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11836 break;
11837 }
11838
11839 case OP_PUSHF:
11840 {
11841 uint32_t cbParm;
11842 uint32_t uMask;
11843 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11844 {
11845 cbParm = 4;
11846 uMask = 0xffffffff;
11847 }
11848 else
11849 {
11850 cbParm = 2;
11851 uMask = 0xffff;
11852 }
11853
11854 /* Get the stack pointer & push the contents of eflags onto the stack. */
11855 RTGCPTR GCPtrStack = 0;
11856 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11857 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11858 if (RT_FAILURE(rc))
11859 {
11860 rc = VERR_EM_INTERPRETER;
11861 break;
11862 }
11863 X86EFLAGS Eflags = pMixedCtx->eflags;
11864 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11865 Eflags.Bits.u1RF = 0;
11866 Eflags.Bits.u1VM = 0;
11867
11868 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
11869 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11870 {
11871 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
11872 rc = VERR_EM_INTERPRETER;
11873 break;
11874 }
11875 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11876 pMixedCtx->esp -= cbParm;
11877 pMixedCtx->esp &= uMask;
11878 pMixedCtx->rip += pDis->cbInstr;
11879 pMixedCtx->eflags.Bits.u1RF = 0;
11880 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11881 | HM_CHANGED_GUEST_RSP
11882 | HM_CHANGED_GUEST_RFLAGS);
11883 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11884 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11885 break;
11886 }
11887
11888 case OP_IRET:
11889 {
11890 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11891 * instruction reference. */
11892 RTGCPTR GCPtrStack = 0;
11893 uint32_t uMask = 0xffff;
11894 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11895 uint16_t aIretFrame[3];
11896 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11897 {
11898 rc = VERR_EM_INTERPRETER;
11899 break;
11900 }
11901 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11902 &GCPtrStack);
11903 if (RT_SUCCESS(rc))
11904 {
11905 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
11906 PGMACCESSORIGIN_HM));
11907 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
11908 }
11909 if (RT_FAILURE(rc))
11910 {
11911 rc = VERR_EM_INTERPRETER;
11912 break;
11913 }
11914 pMixedCtx->eip = 0;
11915 pMixedCtx->ip = aIretFrame[0];
11916 pMixedCtx->cs.Sel = aIretFrame[1];
11917 pMixedCtx->cs.ValidSel = aIretFrame[1];
11918 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11919 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
11920 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11921 pMixedCtx->sp += sizeof(aIretFrame);
11922 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11923 | HM_CHANGED_GUEST_SEGMENT_REGS
11924 | HM_CHANGED_GUEST_RSP
11925 | HM_CHANGED_GUEST_RFLAGS);
11926 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11927 if (fStepping)
11928 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11929 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11930 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11931 break;
11932 }
11933
11934 case OP_INT:
11935 {
11936 uint16_t uVector = pDis->Param1.uValue & 0xff;
11937 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11938 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11939 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11940 break;
11941 }
11942
11943 case OP_INTO:
11944 {
11945 if (pMixedCtx->eflags.Bits.u1OF)
11946 {
11947 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11948 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11949 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11950 }
11951 else
11952 {
11953 pMixedCtx->eflags.Bits.u1RF = 0;
11954 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11955 }
11956 break;
11957 }
11958
11959 default:
11960 {
11961 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
11962 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11963 EMCODETYPE_SUPERVISOR);
11964 rc = VBOXSTRICTRC_VAL(rc2);
11965 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11966 /** @todo We have to set pending-debug exceptions here when the guest is
11967 * single-stepping depending on the instruction that was interpreted. */
11968 Log4(("#GP rc=%Rrc\n", rc));
11969 break;
11970 }
11971 }
11972 }
11973 else
11974 rc = VERR_EM_INTERPRETER;
11975
11976 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11977 ("#GP Unexpected rc=%Rrc\n", rc));
11978 return rc;
11979}
11980
11981
11982#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11983/**
11984 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11985 * the exception reported in the VMX transient structure back into the VM.
11986 *
11987 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11988 * up-to-date.
11989 */
11990static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11991{
11992 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11993
11994 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11995 hmR0VmxCheckExitDueToEventDelivery(). */
11996 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11997 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11998 AssertRCReturn(rc, rc);
11999 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12000
12001#ifdef DEBUG_ramshankar
12002 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12003 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12004 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12005#endif
12006
12007 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12008 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12009 return VINF_SUCCESS;
12010}
12011#endif
12012
12013
12014/**
12015 * VM-exit exception handler for #PF (Page-fault exception).
12016 */
12017static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12018{
12019 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12020 PVM pVM = pVCpu->CTX_SUFF(pVM);
12021 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12022 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12023 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12024 AssertRCReturn(rc, rc);
12025
12026#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12027 if (pVM->hm.s.fNestedPaging)
12028 {
12029 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12030 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12031 {
12032 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12033 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12034 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12035 }
12036 else
12037 {
12038 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12039 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12040 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12041 }
12042 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12043 return rc;
12044 }
12045#else
12046 Assert(!pVM->hm.s.fNestedPaging);
12047 NOREF(pVM);
12048#endif
12049
12050 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12051 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12052 if (pVmxTransient->fVectoringPF)
12053 {
12054 Assert(pVCpu->hm.s.Event.fPending);
12055 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12056 }
12057
12058 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12059 AssertRCReturn(rc, rc);
12060
12061 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12062 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12063
12064 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12065 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12066 (RTGCPTR)pVmxTransient->uExitQualification);
12067
12068 Log4(("#PF: rc=%Rrc\n", rc));
12069 if (rc == VINF_SUCCESS)
12070 {
12071 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12072 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12073 * memory? We don't update the whole state here... */
12074 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12075 | HM_CHANGED_GUEST_RSP
12076 | HM_CHANGED_GUEST_RFLAGS
12077 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12078 TRPMResetTrap(pVCpu);
12079 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12080 return rc;
12081 }
12082
12083 if (rc == VINF_EM_RAW_GUEST_TRAP)
12084 {
12085 if (!pVmxTransient->fVectoringDoublePF)
12086 {
12087 /* It's a guest page fault and needs to be reflected to the guest. */
12088 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12089 TRPMResetTrap(pVCpu);
12090 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12091 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12092 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12093 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12094 }
12095 else
12096 {
12097 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12098 TRPMResetTrap(pVCpu);
12099 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12100 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12101 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12102 }
12103
12104 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12105 return VINF_SUCCESS;
12106 }
12107
12108 TRPMResetTrap(pVCpu);
12109 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12110 return rc;
12111}
12112
12113/** @} */
12114
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette