VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 53611

Last change on this file since 53611 was 53611, checked in by vboxsync, 10 years ago

HMVMXR0.cpp: Use assertions better.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 505.0 KB
Line 
1/* $Id: HMVMXR0.cpp 53611 2014-12-30 20:23:08Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2014 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
41# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
43# define HMVMX_ALWAYS_CHECK_GUEST_STATE
44# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
45# define HMVMX_ALWAYS_TRAP_PF
46# define HMVMX_ALWAYS_SWAP_FPU_STATE
47# define HMVMX_ALWAYS_FLUSH_TLB
48# define HMVMX_ALWAYS_SWAP_EFER
49#endif
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define HMVMX_IS_64BIT_HOST_MODE() (true)
57typedef RTHCUINTREG HMVMXHCUINTREG;
58#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
59extern "C" uint32_t g_fVMXIs64bitHost;
60# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
61typedef uint64_t HMVMXHCUINTREG;
62#else
63# define HMVMX_IS_64BIT_HOST_MODE() (false)
64typedef RTHCUINTREG HMVMXHCUINTREG;
65#endif
66
67/** Use the function table. */
68#define HMVMX_USE_FUNCTION_TABLE
69
70/** Determine which tagged-TLB flush handler to use. */
71#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
72#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
73#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
74#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
75
76/** @name Updated-guest-state flags.
77 * @{ */
78#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
79#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
80#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
81#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
82#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
83#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
84#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
85#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
86#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
87#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
88#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
89#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
90#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
91#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
92#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
93#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
94#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
95#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
96#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
97#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
98#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
99 | HMVMX_UPDATED_GUEST_RSP \
100 | HMVMX_UPDATED_GUEST_RFLAGS \
101 | HMVMX_UPDATED_GUEST_CR0 \
102 | HMVMX_UPDATED_GUEST_CR3 \
103 | HMVMX_UPDATED_GUEST_CR4 \
104 | HMVMX_UPDATED_GUEST_GDTR \
105 | HMVMX_UPDATED_GUEST_IDTR \
106 | HMVMX_UPDATED_GUEST_LDTR \
107 | HMVMX_UPDATED_GUEST_TR \
108 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
109 | HMVMX_UPDATED_GUEST_DEBUG \
110 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
111 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
112 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
113 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
114 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
115 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
116 | HMVMX_UPDATED_GUEST_INTR_STATE \
117 | HMVMX_UPDATED_GUEST_APIC_STATE)
118/** @} */
119
120/** @name
121 * Flags to skip redundant reads of some common VMCS fields that are not part of
122 * the guest-CPU state but are in the transient structure.
123 */
124#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
125#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
131/** @} */
132
133/** @name
134 * States of the VMCS.
135 *
136 * This does not reflect all possible VMCS states but currently only those
137 * needed for maintaining the VMCS consistently even when thread-context hooks
138 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
139 */
140#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
141#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
142#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
143/** @} */
144
145/**
146 * Exception bitmap mask for real-mode guests (real-on-v86).
147 *
148 * We need to intercept all exceptions manually (except #PF). #NM is also
149 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
150 * even in real-mode if we have Nested Paging support.
151 */
152#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
153 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
154 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
155 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
156 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
157 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
158 | RT_BIT(X86_XCPT_XF))
159
160/**
161 * Exception bitmap mask for all contributory exceptions.
162 *
163 * Page fault is deliberately excluded here as it's conditional as to whether
164 * it's contributory or benign. Page faults are handled separately.
165 */
166#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
167 | RT_BIT(X86_XCPT_DE))
168
169/** Maximum VM-instruction error number. */
170#define HMVMX_INSTR_ERROR_MAX 28
171
172/** Profiling macro. */
173#ifdef HM_PROFILE_EXIT_DISPATCH
174# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
176#else
177# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
178# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
179#endif
180
181/** Assert that preemption is disabled or covered by thread-context hooks. */
182#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
184
185/** Assert that we haven't migrated CPUs when thread-context hooks are not
186 * used. */
187#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
188 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
189 ("Illegal migration! Entered on CPU %u Current %u\n", \
190 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
191
192/** Helper macro for VM-exit handlers called unexpectedly. */
193#define HMVMX_RETURN_UNEXPECTED_EXIT() \
194 do { \
195 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
196 return VERR_VMX_UNEXPECTED_EXIT; \
197 } while (0)
198
199
200/*******************************************************************************
201* Structures and Typedefs *
202*******************************************************************************/
203/**
204 * VMX transient state.
205 *
206 * A state structure for holding miscellaneous information across
207 * VMX non-root operation and restored after the transition.
208 */
209typedef struct VMXTRANSIENT
210{
211 /** The host's rflags/eflags. */
212 RTCCUINTREG uEflags;
213#if HC_ARCH_BITS == 32
214 uint32_t u32Alignment0;
215#endif
216 /** The guest's TPR value used for TPR shadowing. */
217 uint8_t u8GuestTpr;
218 /** Alignment. */
219 uint8_t abAlignment0[7];
220
221 /** The basic VM-exit reason. */
222 uint16_t uExitReason;
223 /** Alignment. */
224 uint16_t u16Alignment0;
225 /** The VM-exit interruption error code. */
226 uint32_t uExitIntErrorCode;
227 /** The VM-exit exit code qualification. */
228 uint64_t uExitQualification;
229
230 /** The VM-exit interruption-information field. */
231 uint32_t uExitIntInfo;
232 /** The VM-exit instruction-length field. */
233 uint32_t cbInstr;
234 /** The VM-exit instruction-information field. */
235 union
236 {
237 /** Plain unsigned int representation. */
238 uint32_t u;
239 /** INS and OUTS information. */
240 struct
241 {
242 uint32_t u6Reserved0 : 7;
243 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
244 uint32_t u3AddrSize : 3;
245 uint32_t u5Reserved1 : 5;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 uint32_t uReserved2 : 14;
249 } StrIo;
250 } ExitInstrInfo;
251 /** Whether the VM-entry failed or not. */
252 bool fVMEntryFailed;
253 /** Alignment. */
254 uint8_t abAlignment1[3];
255
256 /** The VM-entry interruption-information field. */
257 uint32_t uEntryIntInfo;
258 /** The VM-entry exception error code field. */
259 uint32_t uEntryXcptErrorCode;
260 /** The VM-entry instruction length field. */
261 uint32_t cbEntryInstr;
262
263 /** IDT-vectoring information field. */
264 uint32_t uIdtVectoringInfo;
265 /** IDT-vectoring error code. */
266 uint32_t uIdtVectoringErrorCode;
267
268 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
269 uint32_t fVmcsFieldsRead;
270
271 /** Whether the guest FPU was active at the time of VM-exit. */
272 bool fWasGuestFPUStateActive;
273 /** Whether the guest debug state was active at the time of VM-exit. */
274 bool fWasGuestDebugStateActive;
275 /** Whether the hyper debug state was active at the time of VM-exit. */
276 bool fWasHyperDebugStateActive;
277 /** Whether TSC-offsetting should be setup before VM-entry. */
278 bool fUpdateTscOffsettingAndPreemptTimer;
279 /** Whether the VM-exit was caused by a page-fault during delivery of a
280 * contributory exception or a page-fault. */
281 bool fVectoringDoublePF;
282 /** Whether the VM-exit was caused by a page-fault during delivery of an
283 * external interrupt or NMI. */
284 bool fVectoringPF;
285} VMXTRANSIENT;
286AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
287AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
288AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
289AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
290AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291/** Pointer to VMX transient state. */
292typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294
295/**
296 * MSR-bitmap read permissions.
297 */
298typedef enum VMXMSREXITREAD
299{
300 /** Reading this MSR causes a VM-exit. */
301 VMXMSREXIT_INTERCEPT_READ = 0xb,
302 /** Reading this MSR does not cause a VM-exit. */
303 VMXMSREXIT_PASSTHRU_READ
304} VMXMSREXITREAD;
305/** Pointer to MSR-bitmap read permissions. */
306typedef VMXMSREXITREAD* PVMXMSREXITREAD;
307
308/**
309 * MSR-bitmap write permissions.
310 */
311typedef enum VMXMSREXITWRITE
312{
313 /** Writing to this MSR causes a VM-exit. */
314 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
315 /** Writing to this MSR does not cause a VM-exit. */
316 VMXMSREXIT_PASSTHRU_WRITE
317} VMXMSREXITWRITE;
318/** Pointer to MSR-bitmap write permissions. */
319typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
320
321
322/**
323 * VMX VM-exit handler.
324 *
325 * @returns VBox status code.
326 * @param pVCpu Pointer to the VMCPU.
327 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
328 * out-of-sync. Make sure to update the required
329 * fields before using them.
330 * @param pVmxTransient Pointer to the VMX-transient structure.
331 */
332#ifndef HMVMX_USE_FUNCTION_TABLE
333typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
334#else
335typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
336/** Pointer to VM-exit handler. */
337typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
338#endif
339
340
341/*******************************************************************************
342* Internal Functions *
343*******************************************************************************/
344static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
345static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
346static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
347 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
348 bool fStepping, uint32_t *puIntState);
349#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
350static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
351#endif
352#ifndef HMVMX_USE_FUNCTION_TABLE
353DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
354# define HMVMX_EXIT_DECL static int
355#else
356# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
357#endif
358DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
359 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
360
361/** @name VM-exit handlers.
362 * @{
363 */
364static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
365static FNVMXEXITHANDLER hmR0VmxExitExtInt;
366static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
367static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
368static FNVMXEXITHANDLER hmR0VmxExitSipi;
369static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
370static FNVMXEXITHANDLER hmR0VmxExitSmi;
371static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
372static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
373static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
374static FNVMXEXITHANDLER hmR0VmxExitCpuid;
375static FNVMXEXITHANDLER hmR0VmxExitGetsec;
376static FNVMXEXITHANDLER hmR0VmxExitHlt;
377static FNVMXEXITHANDLER hmR0VmxExitInvd;
378static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
379static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
380static FNVMXEXITHANDLER hmR0VmxExitVmcall;
381static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
382static FNVMXEXITHANDLER hmR0VmxExitRsm;
383static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
384static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
385static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
386static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
387static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
388static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
389static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
390static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
391static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
392static FNVMXEXITHANDLER hmR0VmxExitMwait;
393static FNVMXEXITHANDLER hmR0VmxExitMtf;
394static FNVMXEXITHANDLER hmR0VmxExitMonitor;
395static FNVMXEXITHANDLER hmR0VmxExitPause;
396static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
397static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
398static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
399static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
400static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
401static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
402static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
403static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
404static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
405static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
406static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
407static FNVMXEXITHANDLER hmR0VmxExitRdrand;
408static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
409/** @} */
410
411static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
412static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
414static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
415static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
417#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
418static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
419#endif
420static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
421
422/*******************************************************************************
423* Global Variables *
424*******************************************************************************/
425#ifdef HMVMX_USE_FUNCTION_TABLE
426
427/**
428 * VMX_EXIT dispatch table.
429 */
430static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
431{
432 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
433 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
434 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
435 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
436 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
437 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
438 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
439 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
440 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
441 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
442 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
443 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
444 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
445 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
446 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
447 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
448 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
449 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
450 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
451 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
452 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
453 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
454 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
455 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
456 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
457 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
458 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
459 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
460 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
461 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
462 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
463 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
464 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
465 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
466 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
467 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
468 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
469 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
470 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
471 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
472 /* 40 UNDEFINED */ hmR0VmxExitPause,
473 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
474 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
475 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
476 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
477 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
478 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
479 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
480 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
481 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
482 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
483 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
484 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
485 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
486 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
487 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
488 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
490 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
491 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
492};
493#endif /* HMVMX_USE_FUNCTION_TABLE */
494
495#ifdef VBOX_STRICT
496static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
497{
498 /* 0 */ "(Not Used)",
499 /* 1 */ "VMCALL executed in VMX root operation.",
500 /* 2 */ "VMCLEAR with invalid physical address.",
501 /* 3 */ "VMCLEAR with VMXON pointer.",
502 /* 4 */ "VMLAUNCH with non-clear VMCS.",
503 /* 5 */ "VMRESUME with non-launched VMCS.",
504 /* 6 */ "VMRESUME after VMXOFF",
505 /* 7 */ "VM-entry with invalid control fields.",
506 /* 8 */ "VM-entry with invalid host state fields.",
507 /* 9 */ "VMPTRLD with invalid physical address.",
508 /* 10 */ "VMPTRLD with VMXON pointer.",
509 /* 11 */ "VMPTRLD with incorrect revision identifier.",
510 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
511 /* 13 */ "VMWRITE to read-only VMCS component.",
512 /* 14 */ "(Not Used)",
513 /* 15 */ "VMXON executed in VMX root operation.",
514 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
515 /* 17 */ "VM-entry with non-launched executing VMCS.",
516 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
517 /* 19 */ "VMCALL with non-clear VMCS.",
518 /* 20 */ "VMCALL with invalid VM-exit control fields.",
519 /* 21 */ "(Not Used)",
520 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
521 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
522 /* 24 */ "VMCALL with invalid SMM-monitor features.",
523 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
524 /* 26 */ "VM-entry with events blocked by MOV SS.",
525 /* 27 */ "(Not Used)",
526 /* 28 */ "Invalid operand to INVEPT/INVVPID."
527};
528#endif /* VBOX_STRICT */
529
530
531
532/**
533 * Updates the VM's last error record. If there was a VMX instruction error,
534 * reads the error data from the VMCS and updates VCPU's last error record as
535 * well.
536 *
537 * @param pVM Pointer to the VM.
538 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
539 * VERR_VMX_UNABLE_TO_START_VM or
540 * VERR_VMX_INVALID_VMCS_FIELD).
541 * @param rc The error code.
542 */
543static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
544{
545 AssertPtr(pVM);
546 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
547 || rc == VERR_VMX_UNABLE_TO_START_VM)
548 {
549 AssertPtrReturnVoid(pVCpu);
550 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
551 }
552 pVM->hm.s.lLastError = rc;
553}
554
555
556/**
557 * Reads the VM-entry interruption-information field from the VMCS into the VMX
558 * transient structure.
559 *
560 * @returns VBox status code.
561 * @param pVmxTransient Pointer to the VMX transient structure.
562 *
563 * @remarks No-long-jump zone!!!
564 */
565DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
566{
567 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
568 AssertRCReturn(rc, rc);
569 return VINF_SUCCESS;
570}
571
572
573/**
574 * Reads the VM-entry exception error code field from the VMCS into
575 * the VMX transient structure.
576 *
577 * @returns VBox status code.
578 * @param pVmxTransient Pointer to the VMX transient structure.
579 *
580 * @remarks No-long-jump zone!!!
581 */
582DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
583{
584 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
585 AssertRCReturn(rc, rc);
586 return VINF_SUCCESS;
587}
588
589
590/**
591 * Reads the VM-entry exception error code field from the VMCS into
592 * the VMX transient structure.
593 *
594 * @returns VBox status code.
595 * @param pVmxTransient Pointer to the VMX transient structure.
596 *
597 * @remarks No-long-jump zone!!!
598 */
599DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
600{
601 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
602 AssertRCReturn(rc, rc);
603 return VINF_SUCCESS;
604}
605
606
607/**
608 * Reads the VM-exit interruption-information field from the VMCS into the VMX
609 * transient structure.
610 *
611 * @returns VBox status code.
612 * @param pVmxTransient Pointer to the VMX transient structure.
613 */
614DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
615{
616 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
617 {
618 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
619 AssertRCReturn(rc, rc);
620 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
621 }
622 return VINF_SUCCESS;
623}
624
625
626/**
627 * Reads the VM-exit interruption error code from the VMCS into the VMX
628 * transient structure.
629 *
630 * @returns VBox status code.
631 * @param pVmxTransient Pointer to the VMX transient structure.
632 */
633DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
634{
635 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
636 {
637 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
638 AssertRCReturn(rc, rc);
639 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
640 }
641 return VINF_SUCCESS;
642}
643
644
645/**
646 * Reads the VM-exit instruction length field from the VMCS into the VMX
647 * transient structure.
648 *
649 * @returns VBox status code.
650 * @param pVCpu Pointer to the VMCPU.
651 * @param pVmxTransient Pointer to the VMX transient structure.
652 */
653DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
654{
655 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
656 {
657 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
658 AssertRCReturn(rc, rc);
659 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
660 }
661 return VINF_SUCCESS;
662}
663
664
665/**
666 * Reads the VM-exit instruction-information field from the VMCS into
667 * the VMX transient structure.
668 *
669 * @returns VBox status code.
670 * @param pVmxTransient Pointer to the VMX transient structure.
671 */
672DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
673{
674 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
675 {
676 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
677 AssertRCReturn(rc, rc);
678 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
679 }
680 return VINF_SUCCESS;
681}
682
683
684/**
685 * Reads the exit code qualification from the VMCS into the VMX transient
686 * structure.
687 *
688 * @returns VBox status code.
689 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
690 * case).
691 * @param pVmxTransient Pointer to the VMX transient structure.
692 */
693DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
694{
695 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
696 {
697 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
698 AssertRCReturn(rc, rc);
699 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
700 }
701 return VINF_SUCCESS;
702}
703
704
705/**
706 * Reads the IDT-vectoring information field from the VMCS into the VMX
707 * transient structure.
708 *
709 * @returns VBox status code.
710 * @param pVmxTransient Pointer to the VMX transient structure.
711 *
712 * @remarks No-long-jump zone!!!
713 */
714DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
715{
716 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
717 {
718 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
719 AssertRCReturn(rc, rc);
720 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
721 }
722 return VINF_SUCCESS;
723}
724
725
726/**
727 * Reads the IDT-vectoring error code from the VMCS into the VMX
728 * transient structure.
729 *
730 * @returns VBox status code.
731 * @param pVmxTransient Pointer to the VMX transient structure.
732 */
733DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
734{
735 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
736 {
737 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
738 AssertRCReturn(rc, rc);
739 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
740 }
741 return VINF_SUCCESS;
742}
743
744
745/**
746 * Enters VMX root mode operation on the current CPU.
747 *
748 * @returns VBox status code.
749 * @param pVM Pointer to the VM (optional, can be NULL, after
750 * a resume).
751 * @param HCPhysCpuPage Physical address of the VMXON region.
752 * @param pvCpuPage Pointer to the VMXON region.
753 */
754static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
755{
756 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
757 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
758 Assert(pvCpuPage);
759 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
760
761 if (pVM)
762 {
763 /* Write the VMCS revision dword to the VMXON region. */
764 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
765 }
766
767 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
768 RTCCUINTREG uEflags = ASMIntDisableFlags();
769
770 /* Enable the VMX bit in CR4 if necessary. */
771 RTCCUINTREG uCr4 = ASMGetCR4();
772 if (!(uCr4 & X86_CR4_VMXE))
773 ASMSetCR4(uCr4 | X86_CR4_VMXE);
774
775 /* Enter VMX root mode. */
776 int rc = VMXEnable(HCPhysCpuPage);
777 if (RT_FAILURE(rc))
778 ASMSetCR4(uCr4);
779
780 /* Restore interrupts. */
781 ASMSetFlags(uEflags);
782 return rc;
783}
784
785
786/**
787 * Exits VMX root mode operation on the current CPU.
788 *
789 * @returns VBox status code.
790 */
791static int hmR0VmxLeaveRootMode(void)
792{
793 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
794
795 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
796 RTCCUINTREG uEflags = ASMIntDisableFlags();
797
798 /* If we're for some reason not in VMX root mode, then don't leave it. */
799 RTCCUINTREG uHostCR4 = ASMGetCR4();
800
801 int rc;
802 if (uHostCR4 & X86_CR4_VMXE)
803 {
804 /* Exit VMX root mode and clear the VMX bit in CR4. */
805 VMXDisable();
806 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
807 rc = VINF_SUCCESS;
808 }
809 else
810 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
811
812 /* Restore interrupts. */
813 ASMSetFlags(uEflags);
814 return rc;
815}
816
817
818/**
819 * Allocates and maps one physically contiguous page. The allocated page is
820 * zero'd out. (Used by various VT-x structures).
821 *
822 * @returns IPRT status code.
823 * @param pMemObj Pointer to the ring-0 memory object.
824 * @param ppVirt Where to store the virtual address of the
825 * allocation.
826 * @param pPhys Where to store the physical address of the
827 * allocation.
828 */
829DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
830{
831 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
832 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
833 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
834
835 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
836 if (RT_FAILURE(rc))
837 return rc;
838 *ppVirt = RTR0MemObjAddress(*pMemObj);
839 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
840 ASMMemZero32(*ppVirt, PAGE_SIZE);
841 return VINF_SUCCESS;
842}
843
844
845/**
846 * Frees and unmaps an allocated physical page.
847 *
848 * @param pMemObj Pointer to the ring-0 memory object.
849 * @param ppVirt Where to re-initialize the virtual address of
850 * allocation as 0.
851 * @param pHCPhys Where to re-initialize the physical address of the
852 * allocation as 0.
853 */
854DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
855{
856 AssertPtr(pMemObj);
857 AssertPtr(ppVirt);
858 AssertPtr(pHCPhys);
859 if (*pMemObj != NIL_RTR0MEMOBJ)
860 {
861 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
862 AssertRC(rc);
863 *pMemObj = NIL_RTR0MEMOBJ;
864 *ppVirt = 0;
865 *pHCPhys = 0;
866 }
867}
868
869
870/**
871 * Worker function to free VT-x related structures.
872 *
873 * @returns IPRT status code.
874 * @param pVM Pointer to the VM.
875 */
876static void hmR0VmxStructsFree(PVM pVM)
877{
878 for (VMCPUID i = 0; i < pVM->cCpus; i++)
879 {
880 PVMCPU pVCpu = &pVM->aCpus[i];
881 AssertPtr(pVCpu);
882
883 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
884 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
885
886 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
887 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
888
889 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
890 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
891 }
892
893 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
894#ifdef VBOX_WITH_CRASHDUMP_MAGIC
895 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
896#endif
897}
898
899
900/**
901 * Worker function to allocate VT-x related VM structures.
902 *
903 * @returns IPRT status code.
904 * @param pVM Pointer to the VM.
905 */
906static int hmR0VmxStructsAlloc(PVM pVM)
907{
908 /*
909 * Initialize members up-front so we can cleanup properly on allocation failure.
910 */
911#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
912 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
913 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
914 pVM->hm.s.vmx.HCPhys##a_Name = 0;
915
916#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
917 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
918 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
919 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
920
921#ifdef VBOX_WITH_CRASHDUMP_MAGIC
922 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
923#endif
924 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
925
926 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
927 for (VMCPUID i = 0; i < pVM->cCpus; i++)
928 {
929 PVMCPU pVCpu = &pVM->aCpus[i];
930 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
931 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
932 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
935 }
936#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
937#undef VMXLOCAL_INIT_VM_MEMOBJ
938
939 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
940 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
941 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
942 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
943
944 /*
945 * Allocate all the VT-x structures.
946 */
947 int rc = VINF_SUCCESS;
948#ifdef VBOX_WITH_CRASHDUMP_MAGIC
949 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
950 if (RT_FAILURE(rc))
951 goto cleanup;
952 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
953 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
954#endif
955
956 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
957 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
958 {
959 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
960 &pVM->hm.s.vmx.HCPhysApicAccess);
961 if (RT_FAILURE(rc))
962 goto cleanup;
963 }
964
965 /*
966 * Initialize per-VCPU VT-x structures.
967 */
968 for (VMCPUID i = 0; i < pVM->cCpus; i++)
969 {
970 PVMCPU pVCpu = &pVM->aCpus[i];
971 AssertPtr(pVCpu);
972
973 /* Allocate the VM control structure (VMCS). */
974 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
975 if (RT_FAILURE(rc))
976 goto cleanup;
977
978 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
979 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
980 {
981 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
982 &pVCpu->hm.s.vmx.HCPhysVirtApic);
983 if (RT_FAILURE(rc))
984 goto cleanup;
985 }
986
987 /*
988 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
989 * transparent accesses of specific MSRs.
990 *
991 * If the condition for enabling MSR bitmaps changes here, don't forget to
992 * update HMIsMsrBitmapsAvailable().
993 */
994 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
995 {
996 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
997 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
998 if (RT_FAILURE(rc))
999 goto cleanup;
1000 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1001 }
1002
1003 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1009 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1010 if (RT_FAILURE(rc))
1011 goto cleanup;
1012 }
1013
1014 return VINF_SUCCESS;
1015
1016cleanup:
1017 hmR0VmxStructsFree(pVM);
1018 return rc;
1019}
1020
1021
1022/**
1023 * Does global VT-x initialization (called during module initialization).
1024 *
1025 * @returns VBox status code.
1026 */
1027VMMR0DECL(int) VMXR0GlobalInit(void)
1028{
1029#ifdef HMVMX_USE_FUNCTION_TABLE
1030 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1031# ifdef VBOX_STRICT
1032 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1033 Assert(g_apfnVMExitHandlers[i]);
1034# endif
1035#endif
1036 return VINF_SUCCESS;
1037}
1038
1039
1040/**
1041 * Does global VT-x termination (called during module termination).
1042 */
1043VMMR0DECL(void) VMXR0GlobalTerm()
1044{
1045 /* Nothing to do currently. */
1046}
1047
1048
1049/**
1050 * Sets up and activates VT-x on the current CPU.
1051 *
1052 * @returns VBox status code.
1053 * @param pCpu Pointer to the global CPU info struct.
1054 * @param pVM Pointer to the VM (can be NULL after a host resume
1055 * operation).
1056 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1057 * fEnabledByHost is true).
1058 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1059 * @a fEnabledByHost is true).
1060 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1061 * enable VT-x on the host.
1062 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1063 */
1064VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1065 void *pvMsrs)
1066{
1067 Assert(pCpu);
1068 Assert(pvMsrs);
1069 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1070
1071 /* Enable VT-x if it's not already enabled by the host. */
1072 if (!fEnabledByHost)
1073 {
1074 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1075 if (RT_FAILURE(rc))
1076 return rc;
1077 }
1078
1079 /*
1080 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1081 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1082 */
1083 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1084 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1085 {
1086 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1087 pCpu->fFlushAsidBeforeUse = false;
1088 }
1089 else
1090 pCpu->fFlushAsidBeforeUse = true;
1091
1092 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1093 ++pCpu->cTlbFlushes;
1094
1095 return VINF_SUCCESS;
1096}
1097
1098
1099/**
1100 * Deactivates VT-x on the current CPU.
1101 *
1102 * @returns VBox status code.
1103 * @param pCpu Pointer to the global CPU info struct.
1104 * @param pvCpuPage Pointer to the VMXON region.
1105 * @param HCPhysCpuPage Physical address of the VMXON region.
1106 *
1107 * @remarks This function should never be called when SUPR0EnableVTx() or
1108 * similar was used to enable VT-x on the host.
1109 */
1110VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1111{
1112 NOREF(pCpu);
1113 NOREF(pvCpuPage);
1114 NOREF(HCPhysCpuPage);
1115
1116 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1117 return hmR0VmxLeaveRootMode();
1118}
1119
1120
1121/**
1122 * Sets the permission bits for the specified MSR in the MSR bitmap.
1123 *
1124 * @param pVCpu Pointer to the VMCPU.
1125 * @param uMSR The MSR value.
1126 * @param enmRead Whether reading this MSR causes a VM-exit.
1127 * @param enmWrite Whether writing this MSR causes a VM-exit.
1128 */
1129static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1130{
1131 int32_t iBit;
1132 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1133
1134 /*
1135 * Layout:
1136 * 0x000 - 0x3ff - Low MSR read bits
1137 * 0x400 - 0x7ff - High MSR read bits
1138 * 0x800 - 0xbff - Low MSR write bits
1139 * 0xc00 - 0xfff - High MSR write bits
1140 */
1141 if (uMsr <= 0x00001FFF)
1142 iBit = uMsr;
1143 else if ( uMsr >= 0xC0000000
1144 && uMsr <= 0xC0001FFF)
1145 {
1146 iBit = (uMsr - 0xC0000000);
1147 pbMsrBitmap += 0x400;
1148 }
1149 else
1150 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1151
1152 Assert(iBit <= 0x1fff);
1153 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1154 ASMBitSet(pbMsrBitmap, iBit);
1155 else
1156 ASMBitClear(pbMsrBitmap, iBit);
1157
1158 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1159 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1160 else
1161 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1162}
1163
1164
1165#ifdef VBOX_STRICT
1166/**
1167 * Gets the permission bits for the specified MSR in the MSR bitmap.
1168 *
1169 * @returns VBox status code.
1170 * @retval VINF_SUCCESS if the specified MSR is found.
1171 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1172 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1173 *
1174 * @param pVCpu Pointer to the VMCPU.
1175 * @param uMsr The MSR.
1176 * @param penmRead Where to store the read permissions.
1177 * @param penmWrite Where to store the write permissions.
1178 */
1179static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1180{
1181 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1182 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1183 int32_t iBit;
1184 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1185
1186 /* See hmR0VmxSetMsrPermission() for the layout. */
1187 if (uMsr <= 0x00001FFF)
1188 iBit = uMsr;
1189 else if ( uMsr >= 0xC0000000
1190 && uMsr <= 0xC0001FFF)
1191 {
1192 iBit = (uMsr - 0xC0000000);
1193 pbMsrBitmap += 0x400;
1194 }
1195 else
1196 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1197
1198 Assert(iBit <= 0x1fff);
1199 if (ASMBitTest(pbMsrBitmap, iBit))
1200 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1201 else
1202 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1203
1204 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1205 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1206 else
1207 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1208 return VINF_SUCCESS;
1209}
1210#endif /* VBOX_STRICT */
1211
1212
1213/**
1214 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1215 * area.
1216 *
1217 * @returns VBox status code.
1218 * @param pVCpu Pointer to the VMCPU.
1219 * @param cMsrs The number of MSRs.
1220 */
1221DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1222{
1223 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1224 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1225 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1226 {
1227 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1228 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1229 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1230 }
1231
1232 /* Update number of guest MSRs to load/store across the world-switch. */
1233 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1234 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1235
1236 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1237 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1238
1239 /* Update the VCPU's copy of the MSR count. */
1240 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1241
1242 return VINF_SUCCESS;
1243}
1244
1245
1246/**
1247 * Adds a new (or updates the value of an existing) guest/host MSR
1248 * pair to be swapped during the world-switch as part of the
1249 * auto-load/store MSR area in the VMCS.
1250 *
1251 * @returns true if the MSR was added -and- its value was updated, false
1252 * otherwise.
1253 * @param pVCpu Pointer to the VMCPU.
1254 * @param uMsr The MSR.
1255 * @param uGuestMsr Value of the guest MSR.
1256 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1257 * necessary.
1258 */
1259static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1260{
1261 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1262 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1263 uint32_t i;
1264 for (i = 0; i < cMsrs; i++)
1265 {
1266 if (pGuestMsr->u32Msr == uMsr)
1267 break;
1268 pGuestMsr++;
1269 }
1270
1271 bool fAdded = false;
1272 if (i == cMsrs)
1273 {
1274 ++cMsrs;
1275 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1276 AssertRC(rc);
1277
1278 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1279 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1280 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1281
1282 fAdded = true;
1283 }
1284
1285 /* Update the MSR values in the auto-load/store MSR area. */
1286 pGuestMsr->u32Msr = uMsr;
1287 pGuestMsr->u64Value = uGuestMsrValue;
1288
1289 /* Create/update the MSR slot in the host MSR area. */
1290 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1291 pHostMsr += i;
1292 pHostMsr->u32Msr = uMsr;
1293
1294 /*
1295 * Update the host MSR only when requested by the caller AND when we're
1296 * adding it to the auto-load/store area. Otherwise, it would have been
1297 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1298 */
1299 bool fUpdatedMsrValue = false;
1300 if ( fAdded
1301 && fUpdateHostMsr)
1302 {
1303 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1304 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1305 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1306 fUpdatedMsrValue = true;
1307 }
1308
1309 return fUpdatedMsrValue;
1310}
1311
1312
1313/**
1314 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1315 * auto-load/store MSR area in the VMCS.
1316 *
1317 * @returns VBox status code.
1318 * @param pVCpu Pointer to the VMCPU.
1319 * @param uMsr The MSR.
1320 */
1321static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1322{
1323 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1324 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1325 for (uint32_t i = 0; i < cMsrs; i++)
1326 {
1327 /* Find the MSR. */
1328 if (pGuestMsr->u32Msr == uMsr)
1329 {
1330 /* If it's the last MSR, simply reduce the count. */
1331 if (i == cMsrs - 1)
1332 {
1333 --cMsrs;
1334 break;
1335 }
1336
1337 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1338 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1339 pLastGuestMsr += cMsrs - 1;
1340 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1341 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1342
1343 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1344 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1345 pLastHostMsr += cMsrs - 1;
1346 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1347 pHostMsr->u64Value = pLastHostMsr->u64Value;
1348 --cMsrs;
1349 break;
1350 }
1351 pGuestMsr++;
1352 }
1353
1354 /* Update the VMCS if the count changed (meaning the MSR was found). */
1355 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1356 {
1357 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1358 AssertRCReturn(rc, rc);
1359
1360 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1361 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1362 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1363
1364 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1365 return VINF_SUCCESS;
1366 }
1367
1368 return VERR_NOT_FOUND;
1369}
1370
1371
1372/**
1373 * Checks if the specified guest MSR is part of the auto-load/store area in
1374 * the VMCS.
1375 *
1376 * @returns true if found, false otherwise.
1377 * @param pVCpu Pointer to the VMCPU.
1378 * @param uMsr The MSR to find.
1379 */
1380static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1381{
1382 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1383 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1384
1385 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1386 {
1387 if (pGuestMsr->u32Msr == uMsr)
1388 return true;
1389 }
1390 return false;
1391}
1392
1393
1394/**
1395 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1396 *
1397 * @param pVCpu Pointer to the VMCPU.
1398 *
1399 * @remarks No-long-jump zone!!!
1400 */
1401static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1402{
1403 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1404 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1405 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1406 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1407
1408 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1409 {
1410 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1411
1412 /*
1413 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1414 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1415 */
1416 if (pHostMsr->u32Msr == MSR_K6_EFER)
1417 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1418 else
1419 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1420 }
1421
1422 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1423}
1424
1425
1426#if HC_ARCH_BITS == 64
1427/**
1428 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1429 * perform lazy restoration of the host MSRs while leaving VT-x.
1430 *
1431 * @param pVCpu Pointer to the VMCPU.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438
1439 /*
1440 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1441 */
1442 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1443 {
1444 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1445 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1446 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1447 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1448 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1449 }
1450}
1451
1452
1453/**
1454 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1455 * lazily while leaving VT-x.
1456 *
1457 * @returns true if it does, false otherwise.
1458 * @param pVCpu Pointer to the VMCPU.
1459 * @param uMsr The MSR to check.
1460 */
1461static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1462{
1463 NOREF(pVCpu);
1464 switch (uMsr)
1465 {
1466 case MSR_K8_LSTAR:
1467 case MSR_K6_STAR:
1468 case MSR_K8_SF_MASK:
1469 case MSR_K8_KERNEL_GS_BASE:
1470 return true;
1471 }
1472 return false;
1473}
1474
1475
1476/**
1477 * Saves a set of guest MSRs back into the guest-CPU context.
1478 *
1479 * @param pVCpu Pointer to the VMCPU.
1480 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1481 * out-of-sync. Make sure to update the required fields
1482 * before using them.
1483 *
1484 * @remarks No-long-jump zone!!!
1485 */
1486static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1487{
1488 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1489 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1490
1491 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1492 {
1493 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1494 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1495 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1496 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1497 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1498 }
1499}
1500
1501
1502/**
1503 * Loads a set of guests MSRs to allow read/passthru to the guest.
1504 *
1505 * The name of this function is slightly confusing. This function does NOT
1506 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1507 * common prefix for functions dealing with "lazy restoration" of the shared
1508 * MSRs.
1509 *
1510 * @param pVCpu Pointer to the VMCPU.
1511 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1512 * out-of-sync. Make sure to update the required fields
1513 * before using them.
1514 *
1515 * @remarks No-long-jump zone!!!
1516 */
1517static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1518{
1519 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1520 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1521
1522#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1523 do { \
1524 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1525 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1526 else \
1527 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1528 } while (0)
1529
1530 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1531 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1532 {
1533 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1534 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1535 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1536 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1537 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1538 }
1539 else
1540 {
1541 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1542 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1543 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1544 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1545 }
1546
1547#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1548}
1549
1550
1551/**
1552 * Performs lazy restoration of the set of host MSRs if they were previously
1553 * loaded with guest MSR values.
1554 *
1555 * @param pVCpu Pointer to the VMCPU.
1556 *
1557 * @remarks No-long-jump zone!!!
1558 * @remarks The guest MSRs should have been saved back into the guest-CPU
1559 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1560 */
1561static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1562{
1563 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1564 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1565
1566 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1567 {
1568 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1569 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1570 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1571 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1572 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1573 }
1574 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1575}
1576#endif /* HC_ARCH_BITS == 64 */
1577
1578
1579/**
1580 * Verifies that our cached values of the VMCS controls are all
1581 * consistent with what's actually present in the VMCS.
1582 *
1583 * @returns VBox status code.
1584 * @param pVCpu Pointer to the VMCPU.
1585 */
1586static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1587{
1588 uint32_t u32Val;
1589 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1590 AssertRCReturn(rc, rc);
1591 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1592 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1593
1594 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1595 AssertRCReturn(rc, rc);
1596 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1597 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1598
1599 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1600 AssertRCReturn(rc, rc);
1601 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1602 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1603
1604 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1605 AssertRCReturn(rc, rc);
1606 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1607 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1608
1609 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1610 {
1611 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1612 AssertRCReturn(rc, rc);
1613 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1614 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1615 }
1616
1617 return VINF_SUCCESS;
1618}
1619
1620
1621#ifdef VBOX_STRICT
1622/**
1623 * Verifies that our cached host EFER value has not changed
1624 * since we cached it.
1625 *
1626 * @param pVCpu Pointer to the VMCPU.
1627 */
1628static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1629{
1630 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1631
1632 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1633 {
1634 uint64_t u64Val;
1635 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1636 AssertRC(rc);
1637
1638 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1639 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1640 }
1641}
1642
1643
1644/**
1645 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1646 * VMCS are correct.
1647 *
1648 * @param pVCpu Pointer to the VMCPU.
1649 */
1650static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1651{
1652 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1653
1654 /* Verify MSR counts in the VMCS are what we think it should be. */
1655 uint32_t cMsrs;
1656 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1657 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1658
1659 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1660 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1661
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1663 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1664
1665 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1666 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1667 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1668 {
1669 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1670 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1671 pGuestMsr->u32Msr, cMsrs));
1672
1673 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1674 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1675 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1676
1677 /* Verify that the permissions are as expected in the MSR bitmap. */
1678 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1679 {
1680 VMXMSREXITREAD enmRead;
1681 VMXMSREXITWRITE enmWrite;
1682 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1683 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1684 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1685 {
1686 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1687 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1688 }
1689 else
1690 {
1691 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1692 pGuestMsr->u32Msr, cMsrs));
1693 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1694 pGuestMsr->u32Msr, cMsrs));
1695 }
1696 }
1697 }
1698}
1699#endif /* VBOX_STRICT */
1700
1701
1702/**
1703 * Flushes the TLB using EPT.
1704 *
1705 * @returns VBox status code.
1706 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1707 * enmFlush).
1708 * @param enmFlush Type of flush.
1709 *
1710 * @remarks Caller is responsible for making sure this function is called only
1711 * when NestedPaging is supported and providing @a enmFlush that is
1712 * supported by the CPU.
1713 * @remarks Can be called with interrupts disabled.
1714 */
1715static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1716{
1717 uint64_t au64Descriptor[2];
1718 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1719 au64Descriptor[0] = 0;
1720 else
1721 {
1722 Assert(pVCpu);
1723 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1724 }
1725 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1726
1727 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1728 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1729 rc));
1730 if ( RT_SUCCESS(rc)
1731 && pVCpu)
1732 {
1733 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1734 }
1735}
1736
1737
1738/**
1739 * Flushes the TLB using VPID.
1740 *
1741 * @returns VBox status code.
1742 * @param pVM Pointer to the VM.
1743 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1744 * enmFlush).
1745 * @param enmFlush Type of flush.
1746 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1747 * on @a enmFlush).
1748 *
1749 * @remarks Can be called with interrupts disabled.
1750 */
1751static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1752{
1753 NOREF(pVM);
1754 AssertPtr(pVM);
1755 Assert(pVM->hm.s.vmx.fVpid);
1756
1757 uint64_t au64Descriptor[2];
1758 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1759 {
1760 au64Descriptor[0] = 0;
1761 au64Descriptor[1] = 0;
1762 }
1763 else
1764 {
1765 AssertPtr(pVCpu);
1766 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1767 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1768 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1769 au64Descriptor[1] = GCPtr;
1770 }
1771
1772 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1773 AssertMsg(rc == VINF_SUCCESS,
1774 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1775 if ( RT_SUCCESS(rc)
1776 && pVCpu)
1777 {
1778 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1779 }
1780}
1781
1782
1783/**
1784 * Invalidates a guest page by guest virtual address. Only relevant for
1785 * EPT/VPID, otherwise there is nothing really to invalidate.
1786 *
1787 * @returns VBox status code.
1788 * @param pVM Pointer to the VM.
1789 * @param pVCpu Pointer to the VMCPU.
1790 * @param GCVirt Guest virtual address of the page to invalidate.
1791 */
1792VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1793{
1794 AssertPtr(pVM);
1795 AssertPtr(pVCpu);
1796 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1797
1798 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1799 if (!fFlushPending)
1800 {
1801 /*
1802 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1803 * See @bugref{6043} and @bugref{6177}.
1804 *
1805 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1806 * function maybe called in a loop with individual addresses.
1807 */
1808 if (pVM->hm.s.vmx.fVpid)
1809 {
1810 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1811 {
1812 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1813 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1814 }
1815 else
1816 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1817 }
1818 else if (pVM->hm.s.fNestedPaging)
1819 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1820 }
1821
1822 return VINF_SUCCESS;
1823}
1824
1825
1826/**
1827 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1828 * otherwise there is nothing really to invalidate.
1829 *
1830 * @returns VBox status code.
1831 * @param pVM Pointer to the VM.
1832 * @param pVCpu Pointer to the VMCPU.
1833 * @param GCPhys Guest physical address of the page to invalidate.
1834 */
1835VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1836{
1837 NOREF(pVM); NOREF(GCPhys);
1838 LogFlowFunc(("%RGp\n", GCPhys));
1839
1840 /*
1841 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1842 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1843 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1844 */
1845 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1846 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1847 return VINF_SUCCESS;
1848}
1849
1850
1851/**
1852 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1853 * case where neither EPT nor VPID is supported by the CPU.
1854 *
1855 * @param pVM Pointer to the VM.
1856 * @param pVCpu Pointer to the VMCPU.
1857 * @param pCpu Pointer to the global HM struct.
1858 *
1859 * @remarks Called with interrupts disabled.
1860 */
1861static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1862{
1863 AssertPtr(pVCpu);
1864 AssertPtr(pCpu);
1865 NOREF(pVM);
1866
1867 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1868
1869 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1870#if 0
1871 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1872 pVCpu->hm.s.TlbShootdown.cPages = 0;
1873#endif
1874
1875 Assert(pCpu->idCpu != NIL_RTCPUID);
1876 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1877 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1878 pVCpu->hm.s.fForceTLBFlush = false;
1879 return;
1880}
1881
1882
1883/**
1884 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1885 *
1886 * @param pVM Pointer to the VM.
1887 * @param pVCpu Pointer to the VMCPU.
1888 * @param pCpu Pointer to the global HM CPU struct.
1889 * @remarks All references to "ASID" in this function pertains to "VPID" in
1890 * Intel's nomenclature. The reason is, to avoid confusion in compare
1891 * statements since the host-CPU copies are named "ASID".
1892 *
1893 * @remarks Called with interrupts disabled.
1894 */
1895static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1896{
1897#ifdef VBOX_WITH_STATISTICS
1898 bool fTlbFlushed = false;
1899# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1900# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1901 if (!fTlbFlushed) \
1902 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1903 } while (0)
1904#else
1905# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1906# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1907#endif
1908
1909 AssertPtr(pVM);
1910 AssertPtr(pCpu);
1911 AssertPtr(pVCpu);
1912 Assert(pCpu->idCpu != NIL_RTCPUID);
1913
1914 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1915 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1916 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1917
1918 /*
1919 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1920 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1921 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1922 */
1923 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1924 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1925 {
1926 ++pCpu->uCurrentAsid;
1927 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1928 {
1929 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1930 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1931 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1932 }
1933
1934 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1935 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1936 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1937
1938 /*
1939 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1940 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1941 */
1942 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1943 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1944 HMVMX_SET_TAGGED_TLB_FLUSHED();
1945 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1946 }
1947
1948 /* Check for explicit TLB shootdowns. */
1949 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1950 {
1951 /*
1952 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1953 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1954 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1955 * but not guest-physical mappings.
1956 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1957 */
1958 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1959 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1960 HMVMX_SET_TAGGED_TLB_FLUSHED();
1961 }
1962
1963 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1964 * where it is commented out. Support individual entry flushing
1965 * someday. */
1966#if 0
1967 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1968 {
1969 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1970
1971 /*
1972 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1973 * as supported by the CPU.
1974 */
1975 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1976 {
1977 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1978 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1979 }
1980 else
1981 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1982
1983 HMVMX_SET_TAGGED_TLB_FLUSHED();
1984 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1985 pVCpu->hm.s.TlbShootdown.cPages = 0;
1986 }
1987#endif
1988
1989 pVCpu->hm.s.fForceTLBFlush = false;
1990
1991 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1992
1993 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1994 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1995 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1996 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1997 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1998 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1999 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2000 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2001 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2002
2003 /* Update VMCS with the VPID. */
2004 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2005 AssertRC(rc);
2006
2007#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2008}
2009
2010
2011/**
2012 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2013 *
2014 * @returns VBox status code.
2015 * @param pVM Pointer to the VM.
2016 * @param pVCpu Pointer to the VMCPU.
2017 * @param pCpu Pointer to the global HM CPU struct.
2018 *
2019 * @remarks Called with interrupts disabled.
2020 */
2021static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2022{
2023 AssertPtr(pVM);
2024 AssertPtr(pVCpu);
2025 AssertPtr(pCpu);
2026 Assert(pCpu->idCpu != NIL_RTCPUID);
2027 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2028 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2029
2030 /*
2031 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2032 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2033 */
2034 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2035 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2036 {
2037 pVCpu->hm.s.fForceTLBFlush = true;
2038 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2039 }
2040
2041 /* Check for explicit TLB shootdown flushes. */
2042 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2043 {
2044 pVCpu->hm.s.fForceTLBFlush = true;
2045 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2046 }
2047
2048 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2049 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2050
2051 if (pVCpu->hm.s.fForceTLBFlush)
2052 {
2053 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2054 pVCpu->hm.s.fForceTLBFlush = false;
2055 }
2056 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2057 * where it is commented out. Support individual entry flushing
2058 * someday. */
2059#if 0
2060 else
2061 {
2062 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2063 {
2064 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2065 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2066 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2067 }
2068 else
2069 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2070
2071 pVCpu->hm.s.TlbShootdown.cPages = 0;
2072 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2073 }
2074#endif
2075}
2076
2077
2078/**
2079 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2080 *
2081 * @returns VBox status code.
2082 * @param pVM Pointer to the VM.
2083 * @param pVCpu Pointer to the VMCPU.
2084 * @param pCpu Pointer to the global HM CPU struct.
2085 *
2086 * @remarks Called with interrupts disabled.
2087 */
2088static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089{
2090 AssertPtr(pVM);
2091 AssertPtr(pVCpu);
2092 AssertPtr(pCpu);
2093 Assert(pCpu->idCpu != NIL_RTCPUID);
2094 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2095 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2096
2097 /*
2098 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2099 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2100 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2101 */
2102 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104 {
2105 pVCpu->hm.s.fForceTLBFlush = true;
2106 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107 }
2108
2109 /* Check for explicit TLB shootdown flushes. */
2110 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111 {
2112 /*
2113 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2114 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2115 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2116 */
2117 pVCpu->hm.s.fForceTLBFlush = true;
2118 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2119 }
2120
2121 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2122 if (pVCpu->hm.s.fForceTLBFlush)
2123 {
2124 ++pCpu->uCurrentAsid;
2125 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2126 {
2127 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2128 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2129 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2130 }
2131
2132 pVCpu->hm.s.fForceTLBFlush = false;
2133 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2134 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2135 if (pCpu->fFlushAsidBeforeUse)
2136 {
2137 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2138 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2139 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2140 {
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2142 pCpu->fFlushAsidBeforeUse = false;
2143 }
2144 else
2145 {
2146 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2147 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2148 }
2149 }
2150 }
2151 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2152 * where it is commented out. Support individual entry flushing
2153 * someday. */
2154#if 0
2155 else
2156 {
2157 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2158 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2159 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2160 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2161
2162 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2163 {
2164 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2165 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2166 {
2167 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2168 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2169 }
2170 else
2171 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2172
2173 pVCpu->hm.s.TlbShootdown.cPages = 0;
2174 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2175 }
2176 else
2177 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2178 }
2179#endif
2180
2181 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2182 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2183 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2184 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2185 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2186 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2187 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2188
2189 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2190 AssertRC(rc);
2191}
2192
2193
2194/**
2195 * Flushes the guest TLB entry based on CPU capabilities.
2196 *
2197 * @param pVCpu Pointer to the VMCPU.
2198 * @param pCpu Pointer to the global HM CPU struct.
2199 */
2200DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2201{
2202#ifdef HMVMX_ALWAYS_FLUSH_TLB
2203 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2204#endif
2205 PVM pVM = pVCpu->CTX_SUFF(pVM);
2206 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2207 {
2208 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2209 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2210 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2211 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2212 default:
2213 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2214 break;
2215 }
2216
2217 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2218 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2219
2220 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2221}
2222
2223
2224/**
2225 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2226 * TLB entries from the host TLB before VM-entry.
2227 *
2228 * @returns VBox status code.
2229 * @param pVM Pointer to the VM.
2230 */
2231static int hmR0VmxSetupTaggedTlb(PVM pVM)
2232{
2233 /*
2234 * Determine optimal flush type for Nested Paging.
2235 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2236 * guest execution (see hmR3InitFinalizeR0()).
2237 */
2238 if (pVM->hm.s.fNestedPaging)
2239 {
2240 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2241 {
2242 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2243 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2244 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2245 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2246 else
2247 {
2248 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2249 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2250 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2251 }
2252
2253 /* Make sure the write-back cacheable memory type for EPT is supported. */
2254 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2255 {
2256 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2257 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2259 }
2260 }
2261 else
2262 {
2263 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2264 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2265 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2266 }
2267 }
2268
2269 /*
2270 * Determine optimal flush type for VPID.
2271 */
2272 if (pVM->hm.s.vmx.fVpid)
2273 {
2274 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2275 {
2276 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2277 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2278 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2279 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2280 else
2281 {
2282 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2283 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2284 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2285 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2286 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2287 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2288 pVM->hm.s.vmx.fVpid = false;
2289 }
2290 }
2291 else
2292 {
2293 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2294 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2295 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2296 pVM->hm.s.vmx.fVpid = false;
2297 }
2298 }
2299
2300 /*
2301 * Setup the handler for flushing tagged-TLBs.
2302 */
2303 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2304 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2305 else if (pVM->hm.s.fNestedPaging)
2306 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2307 else if (pVM->hm.s.vmx.fVpid)
2308 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2309 else
2310 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2311 return VINF_SUCCESS;
2312}
2313
2314
2315/**
2316 * Sets up pin-based VM-execution controls in the VMCS.
2317 *
2318 * @returns VBox status code.
2319 * @param pVM Pointer to the VM.
2320 * @param pVCpu Pointer to the VMCPU.
2321 */
2322static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2323{
2324 AssertPtr(pVM);
2325 AssertPtr(pVCpu);
2326
2327 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2328 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2329
2330 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2331 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2332
2333 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2334 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2335
2336 /* Enable the VMX preemption timer. */
2337 if (pVM->hm.s.vmx.fUsePreemptTimer)
2338 {
2339 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2340 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2341 }
2342
2343 if ((val & zap) != val)
2344 {
2345 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2346 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2347 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2348 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2349 }
2350
2351 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2352 AssertRCReturn(rc, rc);
2353
2354 pVCpu->hm.s.vmx.u32PinCtls = val;
2355 return rc;
2356}
2357
2358
2359/**
2360 * Sets up processor-based VM-execution controls in the VMCS.
2361 *
2362 * @returns VBox status code.
2363 * @param pVM Pointer to the VM.
2364 * @param pVMCPU Pointer to the VMCPU.
2365 */
2366static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2367{
2368 AssertPtr(pVM);
2369 AssertPtr(pVCpu);
2370
2371 int rc = VERR_INTERNAL_ERROR_5;
2372 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2373 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2374
2375 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2376 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2377 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2378 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2379 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2380 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2381 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2382
2383 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2384 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2385 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2386 {
2387 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2388 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2389 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2390 }
2391
2392 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2393 if (!pVM->hm.s.fNestedPaging)
2394 {
2395 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2396 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2397 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2398 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2399 }
2400
2401 /* Use TPR shadowing if supported by the CPU. */
2402 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2403 {
2404 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2405 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2406 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2407 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2408 AssertRCReturn(rc, rc);
2409
2410 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2411 /* CR8 writes cause a VM-exit based on TPR threshold. */
2412 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2413 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2414 }
2415 else
2416 {
2417 /*
2418 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2419 * Set this control only for 64-bit guests.
2420 */
2421 if (pVM->hm.s.fAllow64BitGuests)
2422 {
2423 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2424 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2425 }
2426 }
2427
2428 /* Use MSR-bitmaps if supported by the CPU. */
2429 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2430 {
2431 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2432
2433 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2434 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2435 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2436 AssertRCReturn(rc, rc);
2437
2438 /*
2439 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2440 * automatically using dedicated fields in the VMCS.
2441 */
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447
2448#if HC_ARCH_BITS == 64
2449 /*
2450 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2451 */
2452 if (pVM->hm.s.fAllow64BitGuests)
2453 {
2454 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2455 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2456 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2457 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2458 }
2459#endif
2460 }
2461
2462 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2463 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2464 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2465
2466 if ((val & zap) != val)
2467 {
2468 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2469 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2470 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2471 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2472 }
2473
2474 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2475 AssertRCReturn(rc, rc);
2476
2477 pVCpu->hm.s.vmx.u32ProcCtls = val;
2478
2479 /*
2480 * Secondary processor-based VM-execution controls.
2481 */
2482 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2483 {
2484 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2485 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2486
2487 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2488 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2489
2490 if (pVM->hm.s.fNestedPaging)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2492 else
2493 {
2494 /*
2495 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2496 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2497 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2498 */
2499 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2500 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2501 }
2502
2503 if (pVM->hm.s.vmx.fVpid)
2504 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2505
2506 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2507 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2508
2509 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2510 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2511 * done dynamically. */
2512 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2513 {
2514 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2515 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2516 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2517 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2518 AssertRCReturn(rc, rc);
2519 }
2520
2521 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2522 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2523
2524 if ((val & zap) != val)
2525 {
2526 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2527 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2528 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2529 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2530 }
2531
2532 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2533 AssertRCReturn(rc, rc);
2534
2535 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2536 }
2537 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2538 {
2539 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2540 "available\n"));
2541 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2542 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2543 }
2544
2545 return VINF_SUCCESS;
2546}
2547
2548
2549/**
2550 * Sets up miscellaneous (everything other than Pin & Processor-based
2551 * VM-execution) control fields in the VMCS.
2552 *
2553 * @returns VBox status code.
2554 * @param pVM Pointer to the VM.
2555 * @param pVCpu Pointer to the VMCPU.
2556 */
2557static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2558{
2559 NOREF(pVM);
2560 AssertPtr(pVM);
2561 AssertPtr(pVCpu);
2562
2563 int rc = VERR_GENERAL_FAILURE;
2564
2565 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2566#if 0
2567 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2568 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2569 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2570
2571 /*
2572 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2573 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2574 * We thus use the exception bitmap to control it rather than use both.
2575 */
2576 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2577 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2578
2579 /** @todo Explore possibility of using IO-bitmaps. */
2580 /* All IO & IOIO instructions cause VM-exits. */
2581 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2582 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2583
2584 /* Initialize the MSR-bitmap area. */
2585 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2586 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2587 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2588#endif
2589
2590 /* Setup MSR auto-load/store area. */
2591 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2592 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2593 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2594 AssertRCReturn(rc, rc);
2595 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2596 AssertRCReturn(rc, rc);
2597
2598 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2599 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2600 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2601 AssertRCReturn(rc, rc);
2602
2603 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2604 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2605 AssertRCReturn(rc, rc);
2606
2607 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2608#if 0
2609 /* Setup debug controls */
2610 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2611 AssertRCReturn(rc, rc);
2612 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2613 AssertRCReturn(rc, rc);
2614#endif
2615
2616 return rc;
2617}
2618
2619
2620/**
2621 * Sets up the initial exception bitmap in the VMCS based on static conditions
2622 * (i.e. conditions that cannot ever change after starting the VM).
2623 *
2624 * @returns VBox status code.
2625 * @param pVM Pointer to the VM.
2626 * @param pVCpu Pointer to the VMCPU.
2627 */
2628static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2629{
2630 AssertPtr(pVM);
2631 AssertPtr(pVCpu);
2632
2633 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2634
2635 uint32_t u32XcptBitmap = 0;
2636
2637 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2638 if (!pVM->hm.s.fNestedPaging)
2639 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2640
2641 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2642 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2643 AssertRCReturn(rc, rc);
2644 return rc;
2645}
2646
2647
2648/**
2649 * Sets up the initial guest-state mask. The guest-state mask is consulted
2650 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2651 * for the nested virtualization case (as it would cause a VM-exit).
2652 *
2653 * @param pVCpu Pointer to the VMCPU.
2654 */
2655static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2656{
2657 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2658 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2659 return VINF_SUCCESS;
2660}
2661
2662
2663/**
2664 * Does per-VM VT-x initialization.
2665 *
2666 * @returns VBox status code.
2667 * @param pVM Pointer to the VM.
2668 */
2669VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2670{
2671 LogFlowFunc(("pVM=%p\n", pVM));
2672
2673 int rc = hmR0VmxStructsAlloc(pVM);
2674 if (RT_FAILURE(rc))
2675 {
2676 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2677 return rc;
2678 }
2679
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x termination.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM Pointer to the VM.
2689 */
2690VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2695 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2696 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2697#endif
2698 hmR0VmxStructsFree(pVM);
2699 return VINF_SUCCESS;
2700}
2701
2702
2703/**
2704 * Sets up the VM for execution under VT-x.
2705 * This function is only called once per-VM during initialization.
2706 *
2707 * @returns VBox status code.
2708 * @param pVM Pointer to the VM.
2709 */
2710VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2711{
2712 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2713 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2714
2715 LogFlowFunc(("pVM=%p\n", pVM));
2716
2717 /*
2718 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2719 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2720 */
2721 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2722 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2723 || !pVM->hm.s.vmx.pRealModeTSS))
2724 {
2725 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2726 return VERR_INTERNAL_ERROR;
2727 }
2728
2729#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2730 /*
2731 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2732 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2733 */
2734 if ( pVM->hm.s.fAllow64BitGuests
2735 && !HMVMX_IS_64BIT_HOST_MODE())
2736 {
2737 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2738 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2739 }
2740#endif
2741
2742 /* Initialize these always, see hmR3InitFinalizeR0().*/
2743 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2744 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2745
2746 /* Setup the tagged-TLB flush handlers. */
2747 int rc = hmR0VmxSetupTaggedTlb(pVM);
2748 if (RT_FAILURE(rc))
2749 {
2750 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2751 return rc;
2752 }
2753
2754 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2755 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2756#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2757 if ( HMVMX_IS_64BIT_HOST_MODE()
2758 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2759 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2760 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2761 {
2762 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2763 }
2764#endif
2765
2766 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2767 {
2768 PVMCPU pVCpu = &pVM->aCpus[i];
2769 AssertPtr(pVCpu);
2770 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2771
2772 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2773 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2774
2775 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2776 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2777 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2778
2779 /* Set revision dword at the beginning of the VMCS structure. */
2780 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2781
2782 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2783 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2784 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2785 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2786
2787 /* Load this VMCS as the current VMCS. */
2788 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2789 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2790 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2791
2792 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2793 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2794 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2795
2796 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2797 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2798 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2799
2800 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2813 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2814 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2815 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2816#endif
2817
2818 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2819 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2820 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2821 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2822
2823 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2824
2825 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2826 }
2827
2828 return VINF_SUCCESS;
2829}
2830
2831
2832/**
2833 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2834 * the VMCS.
2835 *
2836 * @returns VBox status code.
2837 * @param pVM Pointer to the VM.
2838 * @param pVCpu Pointer to the VMCPU.
2839 */
2840DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2841{
2842 NOREF(pVM); NOREF(pVCpu);
2843
2844 RTCCUINTREG uReg = ASMGetCR0();
2845 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2846 AssertRCReturn(rc, rc);
2847
2848#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2849 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2850 if (HMVMX_IS_64BIT_HOST_MODE())
2851 {
2852 uint64_t uRegCR3 = HMR0Get64bitCR3();
2853 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2854 }
2855 else
2856#endif
2857 {
2858 uReg = ASMGetCR3();
2859 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2860 }
2861 AssertRCReturn(rc, rc);
2862
2863 uReg = ASMGetCR4();
2864 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2865 AssertRCReturn(rc, rc);
2866 return rc;
2867}
2868
2869
2870#if HC_ARCH_BITS == 64
2871/**
2872 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2873 * requirements. See hmR0VmxSaveHostSegmentRegs().
2874 */
2875# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2876 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2877 { \
2878 bool fValidSelector = true; \
2879 if ((selValue) & X86_SEL_LDT) \
2880 { \
2881 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2882 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2883 } \
2884 if (fValidSelector) \
2885 { \
2886 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2887 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2888 } \
2889 (selValue) = 0; \
2890 }
2891#endif
2892
2893
2894/**
2895 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2896 * the host-state area in the VMCS.
2897 *
2898 * @returns VBox status code.
2899 * @param pVM Pointer to the VM.
2900 * @param pVCpu Pointer to the VMCPU.
2901 */
2902DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2903{
2904 int rc = VERR_INTERNAL_ERROR_5;
2905
2906#if HC_ARCH_BITS == 64
2907 /*
2908 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2909 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2910 */
2911 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2912 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2913#endif
2914
2915 /*
2916 * Host DS, ES, FS and GS segment registers.
2917 */
2918#if HC_ARCH_BITS == 64
2919 RTSEL uSelDS = ASMGetDS();
2920 RTSEL uSelES = ASMGetES();
2921 RTSEL uSelFS = ASMGetFS();
2922 RTSEL uSelGS = ASMGetGS();
2923#else
2924 RTSEL uSelDS = 0;
2925 RTSEL uSelES = 0;
2926 RTSEL uSelFS = 0;
2927 RTSEL uSelGS = 0;
2928#endif
2929
2930 /* Recalculate which host-state bits need to be manually restored. */
2931 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2932
2933 /*
2934 * Host CS and SS segment registers.
2935 */
2936#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2937 RTSEL uSelCS;
2938 RTSEL uSelSS;
2939 if (HMVMX_IS_64BIT_HOST_MODE())
2940 {
2941 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2942 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2943 }
2944 else
2945 {
2946 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2947 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2948 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2949 }
2950#else
2951 RTSEL uSelCS = ASMGetCS();
2952 RTSEL uSelSS = ASMGetSS();
2953#endif
2954
2955 /*
2956 * Host TR segment register.
2957 */
2958 RTSEL uSelTR = ASMGetTR();
2959
2960#if HC_ARCH_BITS == 64
2961 /*
2962 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2963 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2964 */
2965 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2966 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2967 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2968 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2969# undef VMXLOCAL_ADJUST_HOST_SEG
2970#endif
2971
2972 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2973 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2974 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2975 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2976 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2977 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2978 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2979 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2980 Assert(uSelCS);
2981 Assert(uSelTR);
2982
2983 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2984#if 0
2985 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2986 Assert(uSelSS != 0);
2987#endif
2988
2989 /* Write these host selector fields into the host-state area in the VMCS. */
2990 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2991 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2992#if HC_ARCH_BITS == 64
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2994 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2995 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2997#endif
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2999
3000 /*
3001 * Host GDTR and IDTR.
3002 */
3003 RTGDTR Gdtr;
3004 RT_ZERO(Gdtr);
3005#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3006 if (HMVMX_IS_64BIT_HOST_MODE())
3007 {
3008 X86XDTR64 Gdtr64;
3009 X86XDTR64 Idtr64;
3010 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3011 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3012 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3013
3014 Gdtr.cbGdt = Gdtr64.cb;
3015 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3016 }
3017 else
3018#endif
3019 {
3020 RTIDTR Idtr;
3021 ASMGetGDTR(&Gdtr);
3022 ASMGetIDTR(&Idtr);
3023 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3024 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3025
3026#if HC_ARCH_BITS == 64
3027 /*
3028 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3029 * maximum limit (0xffff) on every VM-exit.
3030 */
3031 if (Gdtr.cbGdt != 0xffff)
3032 {
3033 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3034 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3035 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3036 }
3037
3038 /*
3039 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3040 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3041 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3042 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3043 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3044 * hosts where we are pretty sure it won't cause trouble.
3045 */
3046# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3047 if (Idtr.cbIdt < 0x0fff)
3048# else
3049 if (Idtr.cbIdt != 0xffff)
3050# endif
3051 {
3052 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3053 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3054 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3055 }
3056#endif
3057 }
3058
3059 /*
3060 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3061 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3062 */
3063 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3064 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3065 VERR_VMX_INVALID_HOST_STATE);
3066
3067 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3068#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3069 if (HMVMX_IS_64BIT_HOST_MODE())
3070 {
3071 /* We need the 64-bit TR base for hybrid darwin. */
3072 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3073 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3074 }
3075 else
3076#endif
3077 {
3078 uintptr_t uTRBase;
3079#if HC_ARCH_BITS == 64
3080 uTRBase = X86DESC64_BASE(pDesc);
3081
3082 /*
3083 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3084 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3085 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3086 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3087 *
3088 * [1] See Intel spec. 3.5 "System Descriptor Types".
3089 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3090 */
3091 Assert(pDesc->System.u4Type == 11);
3092 if ( pDesc->System.u16LimitLow != 0x67
3093 || pDesc->System.u4LimitHigh)
3094 {
3095 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3096 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3097 if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3098 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3099 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3100
3101 /* Store the GDTR here as we need it while restoring TR. */
3102 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3103 }
3104#else
3105 uTRBase = X86DESC_BASE(pDesc);
3106#endif
3107 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3108 }
3109 AssertRCReturn(rc, rc);
3110
3111 /*
3112 * Host FS base and GS base.
3113 */
3114#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3115 if (HMVMX_IS_64BIT_HOST_MODE())
3116 {
3117 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3118 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3119 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3120 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3121
3122# if HC_ARCH_BITS == 64
3123 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3124 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3125 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3126 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3127 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3128# endif
3129 }
3130#endif
3131 return rc;
3132}
3133
3134
3135/**
3136 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3137 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3138 * the host after every successful VM-exit.
3139 *
3140 * @returns VBox status code.
3141 * @param pVM Pointer to the VM.
3142 * @param pVCpu Pointer to the VMCPU.
3143 *
3144 * @remarks No-long-jump zone!!!
3145 */
3146DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3147{
3148 NOREF(pVM);
3149
3150 AssertPtr(pVCpu);
3151 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3152
3153 int rc = VINF_SUCCESS;
3154#if HC_ARCH_BITS == 64
3155 if (pVM->hm.s.fAllow64BitGuests)
3156 hmR0VmxLazySaveHostMsrs(pVCpu);
3157#endif
3158
3159 /*
3160 * Host Sysenter MSRs.
3161 */
3162 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3163 AssertRCReturn(rc, rc);
3164#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3165 if (HMVMX_IS_64BIT_HOST_MODE())
3166 {
3167 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3168 AssertRCReturn(rc, rc);
3169 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3170 }
3171 else
3172 {
3173 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3174 AssertRCReturn(rc, rc);
3175 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3176 }
3177#elif HC_ARCH_BITS == 32
3178 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3179 AssertRCReturn(rc, rc);
3180 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3181#else
3182 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3183 AssertRCReturn(rc, rc);
3184 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3185#endif
3186 AssertRCReturn(rc, rc);
3187
3188 /*
3189 * Host EFER MSR.
3190 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3191 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3192 */
3193 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3194 {
3195 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3196 AssertRCReturn(rc, rc);
3197 }
3198
3199 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3200 * hmR0VmxLoadGuestExitCtls() !! */
3201
3202 return rc;
3203}
3204
3205
3206/**
3207 * Figures out if we need to swap the EFER MSR which is
3208 * particularly expensive.
3209 *
3210 * We check all relevant bits. For now, that's everything
3211 * besides LMA/LME, as these two bits are handled by VM-entry,
3212 * see hmR0VmxLoadGuestExitCtls() and
3213 * hmR0VMxLoadGuestEntryCtls().
3214 *
3215 * @returns true if we need to load guest EFER, false otherwise.
3216 * @param pVCpu Pointer to the VMCPU.
3217 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3218 * out-of-sync. Make sure to update the required fields
3219 * before using them.
3220 *
3221 * @remarks Requires EFER, CR4.
3222 * @remarks No-long-jump zone!!!
3223 */
3224static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3225{
3226#ifdef HMVMX_ALWAYS_SWAP_EFER
3227 return true;
3228#endif
3229
3230#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3231 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3232 if (CPUMIsGuestInLongMode(pVCpu))
3233 return false;
3234#endif
3235
3236 PVM pVM = pVCpu->CTX_SUFF(pVM);
3237 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3238 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3239
3240 /*
3241 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3242 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3243 */
3244 if ( CPUMIsGuestInLongMode(pVCpu)
3245 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3246 {
3247 return true;
3248 }
3249
3250 /*
3251 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3252 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3253 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3254 */
3255 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3256 && (pMixedCtx->cr0 & X86_CR0_PG)
3257 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3258 {
3259 /* Assert that host is PAE capable. */
3260 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3261 return true;
3262 }
3263
3264 /** @todo Check the latest Intel spec. for any other bits,
3265 * like SMEP/SMAP? */
3266 return false;
3267}
3268
3269
3270/**
3271 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3272 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3273 * controls".
3274 *
3275 * @returns VBox status code.
3276 * @param pVCpu Pointer to the VMCPU.
3277 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3278 * out-of-sync. Make sure to update the required fields
3279 * before using them.
3280 *
3281 * @remarks Requires EFER.
3282 * @remarks No-long-jump zone!!!
3283 */
3284DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3285{
3286 int rc = VINF_SUCCESS;
3287 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3288 {
3289 PVM pVM = pVCpu->CTX_SUFF(pVM);
3290 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3291 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3292
3293 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3294 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3295
3296 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3297 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3298 {
3299 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3300 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3301 }
3302 else
3303 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3304
3305 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3306 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3307 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3308 {
3309 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3310 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3311 }
3312
3313 /*
3314 * The following should -not- be set (since we're not in SMM mode):
3315 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3316 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3317 */
3318
3319 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3320 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3321
3322 if ((val & zap) != val)
3323 {
3324 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3325 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3326 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3327 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3328 }
3329
3330 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3331 AssertRCReturn(rc, rc);
3332
3333 pVCpu->hm.s.vmx.u32EntryCtls = val;
3334 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3335 }
3336 return rc;
3337}
3338
3339
3340/**
3341 * Sets up the VM-exit controls in the VMCS.
3342 *
3343 * @returns VBox status code.
3344 * @param pVM Pointer to the VM.
3345 * @param pVCpu Pointer to the VMCPU.
3346 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3347 * out-of-sync. Make sure to update the required fields
3348 * before using them.
3349 *
3350 * @remarks Requires EFER.
3351 */
3352DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3353{
3354 NOREF(pMixedCtx);
3355
3356 int rc = VINF_SUCCESS;
3357 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3358 {
3359 PVM pVM = pVCpu->CTX_SUFF(pVM);
3360 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3361 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3362
3363 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3364 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3365
3366 /*
3367 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3368 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3369 */
3370#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3371 if (HMVMX_IS_64BIT_HOST_MODE())
3372 {
3373 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3374 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3375 }
3376 else
3377 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3378#else
3379 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3380 {
3381 /* The switcher returns to long mode, EFER is managed by the switcher. */
3382 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3383 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3384 }
3385 else
3386 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3387#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3388
3389 /* If the newer VMCS fields for managing EFER exists, use it. */
3390 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3391 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3392 {
3393 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3394 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3395 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3396 }
3397
3398 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3399 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3400
3401 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3402 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3403 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3404
3405 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3406 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3407
3408 if ((val & zap) != val)
3409 {
3410 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3411 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3412 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3413 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3414 }
3415
3416 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3417 AssertRCReturn(rc, rc);
3418
3419 pVCpu->hm.s.vmx.u32ExitCtls = val;
3420 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3421 }
3422 return rc;
3423}
3424
3425
3426/**
3427 * Loads the guest APIC and related state.
3428 *
3429 * @returns VBox status code.
3430 * @param pVM Pointer to the VM.
3431 * @param pVCpu Pointer to the VMCPU.
3432 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3433 * out-of-sync. Make sure to update the required fields
3434 * before using them.
3435 */
3436DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3437{
3438 NOREF(pMixedCtx);
3439
3440 int rc = VINF_SUCCESS;
3441 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3442 {
3443 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3444 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3445 {
3446 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3447
3448 bool fPendingIntr = false;
3449 uint8_t u8Tpr = 0;
3450 uint8_t u8PendingIntr = 0;
3451 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3452 AssertRCReturn(rc, rc);
3453
3454 /*
3455 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3456 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3457 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3458 * the interrupt when we VM-exit for other reasons.
3459 */
3460 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3461 uint32_t u32TprThreshold = 0;
3462 if (fPendingIntr)
3463 {
3464 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3465 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3466 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3467 if (u8PendingPriority <= u8TprPriority)
3468 u32TprThreshold = u8PendingPriority;
3469 else
3470 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3471 }
3472 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3473
3474 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3475 AssertRCReturn(rc, rc);
3476 }
3477
3478 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3479 }
3480 return rc;
3481}
3482
3483
3484/**
3485 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3486 *
3487 * @returns Guest's interruptibility-state.
3488 * @param pVCpu Pointer to the VMCPU.
3489 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3490 * out-of-sync. Make sure to update the required fields
3491 * before using them.
3492 *
3493 * @remarks No-long-jump zone!!!
3494 */
3495DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3496{
3497 /*
3498 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3499 */
3500 uint32_t uIntrState = 0;
3501 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3502 {
3503 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3504 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3505 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3506 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3507 {
3508 if (pMixedCtx->eflags.Bits.u1IF)
3509 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3510 else
3511 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3512 }
3513 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3514 }
3515
3516 /*
3517 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3518 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3519 * setting this would block host-NMIs and IRET will not clear the blocking.
3520 *
3521 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3522 */
3523 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3524 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3525 {
3526 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3527 }
3528
3529 return uIntrState;
3530}
3531
3532
3533/**
3534 * Loads the guest's interruptibility-state into the guest-state area in the
3535 * VMCS.
3536 *
3537 * @returns VBox status code.
3538 * @param pVCpu Pointer to the VMCPU.
3539 * @param uIntrState The interruptibility-state to set.
3540 */
3541static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3542{
3543 NOREF(pVCpu);
3544 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3545 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3546 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3547 AssertRCReturn(rc, rc);
3548 return rc;
3549}
3550
3551
3552/**
3553 * Loads the guest's RIP into the guest-state area in the VMCS.
3554 *
3555 * @returns VBox status code.
3556 * @param pVCpu Pointer to the VMCPU.
3557 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3558 * out-of-sync. Make sure to update the required fields
3559 * before using them.
3560 *
3561 * @remarks No-long-jump zone!!!
3562 */
3563static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3564{
3565 int rc = VINF_SUCCESS;
3566 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3567 {
3568 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3569 AssertRCReturn(rc, rc);
3570
3571 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3572 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3573 HMCPU_CF_VALUE(pVCpu)));
3574 }
3575 return rc;
3576}
3577
3578
3579/**
3580 * Loads the guest's RSP into the guest-state area in the VMCS.
3581 *
3582 * @returns VBox status code.
3583 * @param pVCpu Pointer to the VMCPU.
3584 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3585 * out-of-sync. Make sure to update the required fields
3586 * before using them.
3587 *
3588 * @remarks No-long-jump zone!!!
3589 */
3590static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3591{
3592 int rc = VINF_SUCCESS;
3593 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3594 {
3595 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3596 AssertRCReturn(rc, rc);
3597
3598 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3599 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3600 }
3601 return rc;
3602}
3603
3604
3605/**
3606 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3607 *
3608 * @returns VBox status code.
3609 * @param pVCpu Pointer to the VMCPU.
3610 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3611 * out-of-sync. Make sure to update the required fields
3612 * before using them.
3613 *
3614 * @remarks No-long-jump zone!!!
3615 */
3616static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3617{
3618 int rc = VINF_SUCCESS;
3619 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3620 {
3621 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3622 Let us assert it as such and use 32-bit VMWRITE. */
3623 Assert(!(pMixedCtx->rflags.u64 >> 32));
3624 X86EFLAGS Eflags = pMixedCtx->eflags;
3625 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3626 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3627
3628 /*
3629 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3630 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3631 */
3632 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3633 {
3634 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3635 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3636 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3637 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3638 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3639 }
3640
3641 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3642 AssertRCReturn(rc, rc);
3643
3644 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3645 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3646 }
3647 return rc;
3648}
3649
3650
3651/**
3652 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3653 *
3654 * @returns VBox status code.
3655 * @param pVCpu Pointer to the VMCPU.
3656 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3657 * out-of-sync. Make sure to update the required fields
3658 * before using them.
3659 *
3660 * @remarks No-long-jump zone!!!
3661 */
3662DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3663{
3664 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3665 AssertRCReturn(rc, rc);
3666 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3667 AssertRCReturn(rc, rc);
3668 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3669 AssertRCReturn(rc, rc);
3670 return rc;
3671}
3672
3673
3674/**
3675 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3676 * CR0 is partially shared with the host and we have to consider the FPU bits.
3677 *
3678 * @returns VBox status code.
3679 * @param pVM Pointer to the VM.
3680 * @param pVCpu Pointer to the VMCPU.
3681 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3682 * out-of-sync. Make sure to update the required fields
3683 * before using them.
3684 *
3685 * @remarks No-long-jump zone!!!
3686 */
3687static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3688{
3689 /*
3690 * Guest CR0.
3691 * Guest FPU.
3692 */
3693 int rc = VINF_SUCCESS;
3694 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3695 {
3696 Assert(!(pMixedCtx->cr0 >> 32));
3697 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3698 PVM pVM = pVCpu->CTX_SUFF(pVM);
3699
3700 /* The guest's view (read access) of its CR0 is unblemished. */
3701 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3702 AssertRCReturn(rc, rc);
3703 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3704
3705 /* Setup VT-x's view of the guest CR0. */
3706 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3707 if (pVM->hm.s.fNestedPaging)
3708 {
3709 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3710 {
3711 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3712 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3713 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3714 }
3715 else
3716 {
3717 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3718 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3719 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3720 }
3721
3722 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3723 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3724 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3725
3726 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3727 AssertRCReturn(rc, rc);
3728 }
3729 else
3730 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3731
3732 /*
3733 * Guest FPU bits.
3734 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3735 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3736 */
3737 u32GuestCR0 |= X86_CR0_NE;
3738 bool fInterceptNM = false;
3739 if (CPUMIsGuestFPUStateActive(pVCpu))
3740 {
3741 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3742 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3743 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3744 }
3745 else
3746 {
3747 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3748 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3749 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3750 }
3751
3752 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3753 bool fInterceptMF = false;
3754 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3755 fInterceptMF = true;
3756
3757 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3758 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3759 {
3760 Assert(PDMVmmDevHeapIsEnabled(pVM));
3761 Assert(pVM->hm.s.vmx.pRealModeTSS);
3762 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3763 fInterceptNM = true;
3764 fInterceptMF = true;
3765 }
3766 else
3767 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3768
3769 if (fInterceptNM)
3770 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3771 else
3772 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3773
3774 if (fInterceptMF)
3775 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3776 else
3777 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3778
3779 /* Additional intercepts for debugging, define these yourself explicitly. */
3780#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3781 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3782 | RT_BIT(X86_XCPT_BP)
3783 | RT_BIT(X86_XCPT_DB)
3784 | RT_BIT(X86_XCPT_DE)
3785 | RT_BIT(X86_XCPT_NM)
3786 | RT_BIT(X86_XCPT_TS)
3787 | RT_BIT(X86_XCPT_UD)
3788 | RT_BIT(X86_XCPT_NP)
3789 | RT_BIT(X86_XCPT_SS)
3790 | RT_BIT(X86_XCPT_GP)
3791 | RT_BIT(X86_XCPT_PF)
3792 | RT_BIT(X86_XCPT_MF)
3793 ;
3794#elif defined(HMVMX_ALWAYS_TRAP_PF)
3795 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3796#endif
3797
3798 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3799
3800 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3801 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3802 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3803 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3804 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3805 else
3806 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3807
3808 u32GuestCR0 |= uSetCR0;
3809 u32GuestCR0 &= uZapCR0;
3810 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3811
3812 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3813 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3814 AssertRCReturn(rc, rc);
3815 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3816 AssertRCReturn(rc, rc);
3817 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3818 uZapCR0));
3819
3820 /*
3821 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3822 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3823 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3824 */
3825 uint32_t u32CR0Mask = 0;
3826 u32CR0Mask = X86_CR0_PE
3827 | X86_CR0_NE
3828 | X86_CR0_WP
3829 | X86_CR0_PG
3830 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3831 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3832 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3833
3834 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3835 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3836 * and @bugref{6944}. */
3837#if 0
3838 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3839 u32CR0Mask &= ~X86_CR0_PE;
3840#endif
3841 if (pVM->hm.s.fNestedPaging)
3842 u32CR0Mask &= ~X86_CR0_WP;
3843
3844 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3845 if (fInterceptNM)
3846 {
3847 u32CR0Mask |= X86_CR0_TS
3848 | X86_CR0_MP;
3849 }
3850
3851 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3852 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3853 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3854 AssertRCReturn(rc, rc);
3855 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3856
3857 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3858 }
3859 return rc;
3860}
3861
3862
3863/**
3864 * Loads the guest control registers (CR3, CR4) into the guest-state area
3865 * in the VMCS.
3866 *
3867 * @returns VBox status code.
3868 * @param pVM Pointer to the VM.
3869 * @param pVCpu Pointer to the VMCPU.
3870 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3871 * out-of-sync. Make sure to update the required fields
3872 * before using them.
3873 *
3874 * @remarks No-long-jump zone!!!
3875 */
3876static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3877{
3878 int rc = VINF_SUCCESS;
3879 PVM pVM = pVCpu->CTX_SUFF(pVM);
3880
3881 /*
3882 * Guest CR2.
3883 * It's always loaded in the assembler code. Nothing to do here.
3884 */
3885
3886 /*
3887 * Guest CR3.
3888 */
3889 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3890 {
3891 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3892 if (pVM->hm.s.fNestedPaging)
3893 {
3894 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3895
3896 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3897 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3898 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3899 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3900
3901 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3902 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3903 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3904
3905 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3906 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3907 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3908 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3909
3910 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3911 AssertRCReturn(rc, rc);
3912 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3913
3914 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3915 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3916 {
3917 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3918 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3919 {
3920 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3921 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3922 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3923 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3924 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3925 }
3926
3927 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3928 have Unrestricted Execution to handle the guest when it's not using paging. */
3929 GCPhysGuestCR3 = pMixedCtx->cr3;
3930 }
3931 else
3932 {
3933 /*
3934 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3935 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3936 * EPT takes care of translating it to host-physical addresses.
3937 */
3938 RTGCPHYS GCPhys;
3939 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3940 Assert(PDMVmmDevHeapIsEnabled(pVM));
3941
3942 /* We obtain it here every time as the guest could have relocated this PCI region. */
3943 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3944 AssertRCReturn(rc, rc);
3945
3946 GCPhysGuestCR3 = GCPhys;
3947 }
3948
3949 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3950 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3951 }
3952 else
3953 {
3954 /* Non-nested paging case, just use the hypervisor's CR3. */
3955 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3956
3957 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3958 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3959 }
3960 AssertRCReturn(rc, rc);
3961
3962 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3963 }
3964
3965 /*
3966 * Guest CR4.
3967 */
3968 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3969 {
3970 Assert(!(pMixedCtx->cr4 >> 32));
3971 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3972
3973 /* The guest's view of its CR4 is unblemished. */
3974 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3975 AssertRCReturn(rc, rc);
3976 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3977
3978 /* Setup VT-x's view of the guest CR4. */
3979 /*
3980 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3981 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3982 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3983 */
3984 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3985 {
3986 Assert(pVM->hm.s.vmx.pRealModeTSS);
3987 Assert(PDMVmmDevHeapIsEnabled(pVM));
3988 u32GuestCR4 &= ~X86_CR4_VME;
3989 }
3990
3991 if (pVM->hm.s.fNestedPaging)
3992 {
3993 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3994 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3995 {
3996 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3997 u32GuestCR4 |= X86_CR4_PSE;
3998 /* Our identity mapping is a 32-bit page directory. */
3999 u32GuestCR4 &= ~X86_CR4_PAE;
4000 }
4001 /* else use guest CR4.*/
4002 }
4003 else
4004 {
4005 /*
4006 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4007 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4008 */
4009 switch (pVCpu->hm.s.enmShadowMode)
4010 {
4011 case PGMMODE_REAL: /* Real-mode. */
4012 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4013 case PGMMODE_32_BIT: /* 32-bit paging. */
4014 {
4015 u32GuestCR4 &= ~X86_CR4_PAE;
4016 break;
4017 }
4018
4019 case PGMMODE_PAE: /* PAE paging. */
4020 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4021 {
4022 u32GuestCR4 |= X86_CR4_PAE;
4023 break;
4024 }
4025
4026 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4027 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4028#ifdef VBOX_ENABLE_64_BITS_GUESTS
4029 break;
4030#endif
4031 default:
4032 AssertFailed();
4033 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4034 }
4035 }
4036
4037 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4038 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4039 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4040 u32GuestCR4 |= uSetCR4;
4041 u32GuestCR4 &= uZapCR4;
4042
4043 /* Write VT-x's view of the guest CR4 into the VMCS. */
4044 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4045 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4046 AssertRCReturn(rc, rc);
4047
4048 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4049 uint32_t u32CR4Mask = 0;
4050 u32CR4Mask = X86_CR4_VME
4051 | X86_CR4_PAE
4052 | X86_CR4_PGE
4053 | X86_CR4_PSE
4054 | X86_CR4_VMXE;
4055 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4056 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4057 AssertRCReturn(rc, rc);
4058
4059 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4060 }
4061 return rc;
4062}
4063
4064
4065/**
4066 * Loads the guest debug registers into the guest-state area in the VMCS.
4067 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4068 *
4069 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4070 *
4071 * @returns VBox status code.
4072 * @param pVCpu Pointer to the VMCPU.
4073 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4074 * out-of-sync. Make sure to update the required fields
4075 * before using them.
4076 *
4077 * @remarks No-long-jump zone!!!
4078 */
4079static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4080{
4081 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4082 return VINF_SUCCESS;
4083
4084#ifdef VBOX_STRICT
4085 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4086 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4087 {
4088 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4089 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4090 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4091 }
4092#endif
4093
4094 int rc;
4095 PVM pVM = pVCpu->CTX_SUFF(pVM);
4096 bool fInterceptDB = false;
4097 bool fInterceptMovDRx = false;
4098 if ( pVCpu->hm.s.fSingleInstruction
4099 || DBGFIsStepping(pVCpu))
4100 {
4101 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4102 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4103 {
4104 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4105 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4106 AssertRCReturn(rc, rc);
4107 Assert(fInterceptDB == false);
4108 }
4109 else
4110 {
4111 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4112 pVCpu->hm.s.fClearTrapFlag = true;
4113 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4114 fInterceptDB = true;
4115 }
4116 }
4117
4118 if ( fInterceptDB
4119 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4120 {
4121 /*
4122 * Use the combined guest and host DRx values found in the hypervisor
4123 * register set because the debugger has breakpoints active or someone
4124 * is single stepping on the host side without a monitor trap flag.
4125 *
4126 * Note! DBGF expects a clean DR6 state before executing guest code.
4127 */
4128#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4129 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4130 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4131 {
4132 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4133 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4134 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4135 }
4136 else
4137#endif
4138 if (!CPUMIsHyperDebugStateActive(pVCpu))
4139 {
4140 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4141 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4142 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4143 }
4144
4145 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4146 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4147 AssertRCReturn(rc, rc);
4148
4149 pVCpu->hm.s.fUsingHyperDR7 = true;
4150 fInterceptDB = true;
4151 fInterceptMovDRx = true;
4152 }
4153 else
4154 {
4155 /*
4156 * If the guest has enabled debug registers, we need to load them prior to
4157 * executing guest code so they'll trigger at the right time.
4158 */
4159 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4160 {
4161#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4162 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4163 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4164 {
4165 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4166 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4167 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4168 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4169 }
4170 else
4171#endif
4172 if (!CPUMIsGuestDebugStateActive(pVCpu))
4173 {
4174 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4175 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4176 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4177 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4178 }
4179 Assert(!fInterceptDB);
4180 Assert(!fInterceptMovDRx);
4181 }
4182 /*
4183 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4184 * must intercept #DB in order to maintain a correct DR6 guest value.
4185 */
4186#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4187 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4188 && !CPUMIsGuestDebugStateActive(pVCpu))
4189#else
4190 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4191#endif
4192 {
4193 fInterceptMovDRx = true;
4194 fInterceptDB = true;
4195 }
4196
4197 /* Update guest DR7. */
4198 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4199 AssertRCReturn(rc, rc);
4200
4201 pVCpu->hm.s.fUsingHyperDR7 = false;
4202 }
4203
4204 /*
4205 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4206 */
4207 if (fInterceptDB)
4208 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4209 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4210 {
4211#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4212 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4213#endif
4214 }
4215 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4216 AssertRCReturn(rc, rc);
4217
4218 /*
4219 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4220 */
4221 if (fInterceptMovDRx)
4222 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4223 else
4224 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4225 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4226 AssertRCReturn(rc, rc);
4227
4228 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4229 return VINF_SUCCESS;
4230}
4231
4232
4233#ifdef VBOX_STRICT
4234/**
4235 * Strict function to validate segment registers.
4236 *
4237 * @remarks ASSUMES CR0 is up to date.
4238 */
4239static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4240{
4241 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4242 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4243 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4244 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4245 && ( !CPUMIsGuestInRealModeEx(pCtx)
4246 && !CPUMIsGuestInV86ModeEx(pCtx)))
4247 {
4248 /* Protected mode checks */
4249 /* CS */
4250 Assert(pCtx->cs.Attr.n.u1Present);
4251 Assert(!(pCtx->cs.Attr.u & 0xf00));
4252 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4253 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4254 || !(pCtx->cs.Attr.n.u1Granularity));
4255 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4256 || (pCtx->cs.Attr.n.u1Granularity));
4257 /* CS cannot be loaded with NULL in protected mode. */
4258 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4259 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4260 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4261 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4262 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4263 else
4264 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4265 /* SS */
4266 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4267 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4268 if ( !(pCtx->cr0 & X86_CR0_PE)
4269 || pCtx->cs.Attr.n.u4Type == 3)
4270 {
4271 Assert(!pCtx->ss.Attr.n.u2Dpl);
4272 }
4273 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4274 {
4275 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4276 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4277 Assert(pCtx->ss.Attr.n.u1Present);
4278 Assert(!(pCtx->ss.Attr.u & 0xf00));
4279 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4280 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4281 || !(pCtx->ss.Attr.n.u1Granularity));
4282 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4283 || (pCtx->ss.Attr.n.u1Granularity));
4284 }
4285 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4286 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4287 {
4288 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4289 Assert(pCtx->ds.Attr.n.u1Present);
4290 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4291 Assert(!(pCtx->ds.Attr.u & 0xf00));
4292 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4293 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4294 || !(pCtx->ds.Attr.n.u1Granularity));
4295 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4296 || (pCtx->ds.Attr.n.u1Granularity));
4297 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4298 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4299 }
4300 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4301 {
4302 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4303 Assert(pCtx->es.Attr.n.u1Present);
4304 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4305 Assert(!(pCtx->es.Attr.u & 0xf00));
4306 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4307 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4308 || !(pCtx->es.Attr.n.u1Granularity));
4309 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4310 || (pCtx->es.Attr.n.u1Granularity));
4311 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4312 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4313 }
4314 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4315 {
4316 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4317 Assert(pCtx->fs.Attr.n.u1Present);
4318 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4319 Assert(!(pCtx->fs.Attr.u & 0xf00));
4320 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4321 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4322 || !(pCtx->fs.Attr.n.u1Granularity));
4323 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4324 || (pCtx->fs.Attr.n.u1Granularity));
4325 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4326 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4327 }
4328 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4329 {
4330 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4331 Assert(pCtx->gs.Attr.n.u1Present);
4332 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4333 Assert(!(pCtx->gs.Attr.u & 0xf00));
4334 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4335 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4336 || !(pCtx->gs.Attr.n.u1Granularity));
4337 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4338 || (pCtx->gs.Attr.n.u1Granularity));
4339 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4340 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4341 }
4342 /* 64-bit capable CPUs. */
4343# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4344 if (HMVMX_IS_64BIT_HOST_MODE())
4345 {
4346 Assert(!(pCtx->cs.u64Base >> 32));
4347 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4348 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4349 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4350 }
4351# endif
4352 }
4353 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4354 || ( CPUMIsGuestInRealModeEx(pCtx)
4355 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4356 {
4357 /* Real and v86 mode checks. */
4358 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4359 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4360 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4361 {
4362 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4363 }
4364 else
4365 {
4366 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4367 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4368 }
4369
4370 /* CS */
4371 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4372 Assert(pCtx->cs.u32Limit == 0xffff);
4373 Assert(u32CSAttr == 0xf3);
4374 /* SS */
4375 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4376 Assert(pCtx->ss.u32Limit == 0xffff);
4377 Assert(u32SSAttr == 0xf3);
4378 /* DS */
4379 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4380 Assert(pCtx->ds.u32Limit == 0xffff);
4381 Assert(u32DSAttr == 0xf3);
4382 /* ES */
4383 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4384 Assert(pCtx->es.u32Limit == 0xffff);
4385 Assert(u32ESAttr == 0xf3);
4386 /* FS */
4387 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4388 Assert(pCtx->fs.u32Limit == 0xffff);
4389 Assert(u32FSAttr == 0xf3);
4390 /* GS */
4391 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4392 Assert(pCtx->gs.u32Limit == 0xffff);
4393 Assert(u32GSAttr == 0xf3);
4394 /* 64-bit capable CPUs. */
4395# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4396 if (HMVMX_IS_64BIT_HOST_MODE())
4397 {
4398 Assert(!(pCtx->cs.u64Base >> 32));
4399 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4400 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4401 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4402 }
4403# endif
4404 }
4405}
4406#endif /* VBOX_STRICT */
4407
4408
4409/**
4410 * Writes a guest segment register into the guest-state area in the VMCS.
4411 *
4412 * @returns VBox status code.
4413 * @param pVCpu Pointer to the VMCPU.
4414 * @param idxSel Index of the selector in the VMCS.
4415 * @param idxLimit Index of the segment limit in the VMCS.
4416 * @param idxBase Index of the segment base in the VMCS.
4417 * @param idxAccess Index of the access rights of the segment in the VMCS.
4418 * @param pSelReg Pointer to the segment selector.
4419 *
4420 * @remarks No-long-jump zone!!!
4421 */
4422static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4423 uint32_t idxAccess, PCPUMSELREG pSelReg)
4424{
4425 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4426 AssertRCReturn(rc, rc);
4427 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4428 AssertRCReturn(rc, rc);
4429 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4430 AssertRCReturn(rc, rc);
4431
4432 uint32_t u32Access = pSelReg->Attr.u;
4433 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4434 {
4435 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4436 u32Access = 0xf3;
4437 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4438 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4439 }
4440 else
4441 {
4442 /*
4443 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4444 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4445 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4446 * loaded in protected-mode have their attribute as 0.
4447 */
4448 if (!u32Access)
4449 u32Access = X86DESCATTR_UNUSABLE;
4450 }
4451
4452 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4453 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4454 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4455
4456 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4457 AssertRCReturn(rc, rc);
4458 return rc;
4459}
4460
4461
4462/**
4463 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4464 * into the guest-state area in the VMCS.
4465 *
4466 * @returns VBox status code.
4467 * @param pVM Pointer to the VM.
4468 * @param pVCPU Pointer to the VMCPU.
4469 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4470 * out-of-sync. Make sure to update the required fields
4471 * before using them.
4472 *
4473 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4474 * @remarks No-long-jump zone!!!
4475 */
4476static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4477{
4478 int rc = VERR_INTERNAL_ERROR_5;
4479 PVM pVM = pVCpu->CTX_SUFF(pVM);
4480
4481 /*
4482 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4483 */
4484 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4485 {
4486 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4487 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4488 {
4489 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4490 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4491 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4492 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4493 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4494 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4495 }
4496
4497#ifdef VBOX_WITH_REM
4498 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4499 {
4500 Assert(pVM->hm.s.vmx.pRealModeTSS);
4501 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4502 if ( pVCpu->hm.s.vmx.fWasInRealMode
4503 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4504 {
4505 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4506 in real-mode (e.g. OpenBSD 4.0) */
4507 REMFlushTBs(pVM);
4508 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4509 pVCpu->hm.s.vmx.fWasInRealMode = false;
4510 }
4511 }
4512#endif
4513 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4514 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4515 AssertRCReturn(rc, rc);
4516 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4517 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4518 AssertRCReturn(rc, rc);
4519 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4520 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4521 AssertRCReturn(rc, rc);
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4523 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4526 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4529 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4530 AssertRCReturn(rc, rc);
4531
4532#ifdef VBOX_STRICT
4533 /* Validate. */
4534 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4535#endif
4536
4537 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4538 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4539 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4540 }
4541
4542 /*
4543 * Guest TR.
4544 */
4545 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4546 {
4547 /*
4548 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4549 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4550 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4551 */
4552 uint16_t u16Sel = 0;
4553 uint32_t u32Limit = 0;
4554 uint64_t u64Base = 0;
4555 uint32_t u32AccessRights = 0;
4556
4557 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4558 {
4559 u16Sel = pMixedCtx->tr.Sel;
4560 u32Limit = pMixedCtx->tr.u32Limit;
4561 u64Base = pMixedCtx->tr.u64Base;
4562 u32AccessRights = pMixedCtx->tr.Attr.u;
4563 }
4564 else
4565 {
4566 Assert(pVM->hm.s.vmx.pRealModeTSS);
4567 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4568
4569 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4570 RTGCPHYS GCPhys;
4571 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4572 AssertRCReturn(rc, rc);
4573
4574 X86DESCATTR DescAttr;
4575 DescAttr.u = 0;
4576 DescAttr.n.u1Present = 1;
4577 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4578
4579 u16Sel = 0;
4580 u32Limit = HM_VTX_TSS_SIZE;
4581 u64Base = GCPhys; /* in real-mode phys = virt. */
4582 u32AccessRights = DescAttr.u;
4583 }
4584
4585 /* Validate. */
4586 Assert(!(u16Sel & RT_BIT(2)));
4587 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4588 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4589 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4590 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4591 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4592 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4593 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4594 Assert( (u32Limit & 0xfff) == 0xfff
4595 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4596 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4597 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4598
4599 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4600 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4601 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4602 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4603
4604 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4605 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4606 }
4607
4608 /*
4609 * Guest GDTR.
4610 */
4611 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4612 {
4613 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4614 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4615
4616 /* Validate. */
4617 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4618
4619 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4620 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4621 }
4622
4623 /*
4624 * Guest LDTR.
4625 */
4626 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4627 {
4628 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4629 uint32_t u32Access = 0;
4630 if (!pMixedCtx->ldtr.Attr.u)
4631 u32Access = X86DESCATTR_UNUSABLE;
4632 else
4633 u32Access = pMixedCtx->ldtr.Attr.u;
4634
4635 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4636 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4637 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4638 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4639
4640 /* Validate. */
4641 if (!(u32Access & X86DESCATTR_UNUSABLE))
4642 {
4643 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4644 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4645 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4646 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4647 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4648 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4649 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4650 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4651 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4652 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4653 }
4654
4655 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4656 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4657 }
4658
4659 /*
4660 * Guest IDTR.
4661 */
4662 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4663 {
4664 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4665 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4666
4667 /* Validate. */
4668 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4669
4670 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4671 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4672 }
4673
4674 return VINF_SUCCESS;
4675}
4676
4677
4678/**
4679 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4680 * areas. These MSRs will automatically be loaded to the host CPU on every
4681 * successful VM-entry and stored from the host CPU on every successful VM-exit.
4682 *
4683 * This also creates/updates MSR slots for the host MSRs. The actual host
4684 * MSR values are -not- updated here for performance reasons. See
4685 * hmR0VmxSaveHostMsrs().
4686 *
4687 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4688 *
4689 * @returns VBox status code.
4690 * @param pVCpu Pointer to the VMCPU.
4691 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4692 * out-of-sync. Make sure to update the required fields
4693 * before using them.
4694 *
4695 * @remarks No-long-jump zone!!!
4696 */
4697static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4698{
4699 AssertPtr(pVCpu);
4700 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4701
4702 /*
4703 * MSRs that we use the auto-load/store MSR area in the VMCS.
4704 */
4705 PVM pVM = pVCpu->CTX_SUFF(pVM);
4706 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4707 {
4708 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4709#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4710 if (pVM->hm.s.fAllow64BitGuests)
4711 {
4712 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4713 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4714 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4715 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4716# ifdef DEBUG
4717 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4718 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4719 {
4720 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4721 pMsr->u64Value));
4722 }
4723# endif
4724 }
4725#endif
4726 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4727 }
4728
4729 /*
4730 * Guest Sysenter MSRs.
4731 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4732 * VM-exits on WRMSRs for these MSRs.
4733 */
4734 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4735 {
4736 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4737 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4738 }
4739
4740 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4741 {
4742 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4743 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4744 }
4745
4746 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4747 {
4748 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4749 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4750 }
4751
4752 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4753 {
4754 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4755 {
4756 /*
4757 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4758 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4759 */
4760 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4761 {
4762 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4763 AssertRCReturn(rc,rc);
4764 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4765 }
4766 else
4767 {
4768 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4769 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4770 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4771 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4772 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4773 }
4774 }
4775 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4776 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4777 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4778 }
4779
4780 return VINF_SUCCESS;
4781}
4782
4783
4784/**
4785 * Loads the guest activity state into the guest-state area in the VMCS.
4786 *
4787 * @returns VBox status code.
4788 * @param pVCpu Pointer to the VMCPU.
4789 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4790 * out-of-sync. Make sure to update the required fields
4791 * before using them.
4792 *
4793 * @remarks No-long-jump zone!!!
4794 */
4795static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4796{
4797 NOREF(pCtx);
4798 /** @todo See if we can make use of other states, e.g.
4799 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4800 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4801 {
4802 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4803 AssertRCReturn(rc, rc);
4804
4805 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4806 }
4807 return VINF_SUCCESS;
4808}
4809
4810
4811/**
4812 * Sets up the appropriate function to run guest code.
4813 *
4814 * @returns VBox status code.
4815 * @param pVCpu Pointer to the VMCPU.
4816 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4817 * out-of-sync. Make sure to update the required fields
4818 * before using them.
4819 *
4820 * @remarks No-long-jump zone!!!
4821 */
4822static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4823{
4824 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4825 {
4826#ifndef VBOX_ENABLE_64_BITS_GUESTS
4827 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4828#endif
4829 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4830#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4831 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4832 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4833 {
4834 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4835 {
4836 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4837 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4838 | HM_CHANGED_VMX_EXIT_CTLS
4839 | HM_CHANGED_VMX_ENTRY_CTLS
4840 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4841 }
4842 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4843 }
4844#else
4845 /* 64-bit host or hybrid host. */
4846 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4847#endif
4848 }
4849 else
4850 {
4851 /* Guest is not in long mode, use the 32-bit handler. */
4852#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4853 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4854 {
4855 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4856 {
4857 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4858 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4859 | HM_CHANGED_VMX_EXIT_CTLS
4860 | HM_CHANGED_VMX_ENTRY_CTLS
4861 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4862 }
4863 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4864 }
4865#else
4866 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4867#endif
4868 }
4869 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4870 return VINF_SUCCESS;
4871}
4872
4873
4874/**
4875 * Wrapper for running the guest code in VT-x.
4876 *
4877 * @returns VBox strict status code.
4878 * @param pVM Pointer to the VM.
4879 * @param pVCpu Pointer to the VMCPU.
4880 * @param pCtx Pointer to the guest-CPU context.
4881 *
4882 * @remarks No-long-jump zone!!!
4883 */
4884DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4885{
4886 /*
4887 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4888 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4889 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4890 */
4891 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4892 /** @todo Add stats for resume vs launch. */
4893#ifdef VBOX_WITH_KERNEL_USING_XMM
4894 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4895#else
4896 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4897#endif
4898}
4899
4900
4901/**
4902 * Reports world-switch error and dumps some useful debug info.
4903 *
4904 * @param pVM Pointer to the VM.
4905 * @param pVCpu Pointer to the VMCPU.
4906 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4907 * @param pCtx Pointer to the guest-CPU context.
4908 * @param pVmxTransient Pointer to the VMX transient structure (only
4909 * exitReason updated).
4910 */
4911static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4912{
4913 Assert(pVM);
4914 Assert(pVCpu);
4915 Assert(pCtx);
4916 Assert(pVmxTransient);
4917 HMVMX_ASSERT_PREEMPT_SAFE();
4918
4919 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4920 switch (rcVMRun)
4921 {
4922 case VERR_VMX_INVALID_VMXON_PTR:
4923 AssertFailed();
4924 break;
4925 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4926 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4927 {
4928 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4929 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4930 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4931 AssertRC(rc);
4932
4933 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4934 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4935 Cannot do it here as we may have been long preempted. */
4936
4937#ifdef VBOX_STRICT
4938 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4939 pVmxTransient->uExitReason));
4940 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4941 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4942 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4943 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4944 else
4945 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4946 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4947 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4948
4949 /* VMX control bits. */
4950 uint32_t u32Val;
4951 uint64_t u64Val;
4952 HMVMXHCUINTREG uHCReg;
4953 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4954 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4955 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4956 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4957 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4958 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4959 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4960 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4961 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4962 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4963 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4964 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4965 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4966 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4967 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4968 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4969 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4970 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4971 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4972 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4973 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4974 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4975 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4976 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4977 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4978 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4979 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4980 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4981 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4982 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4983 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4984 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4985 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4986 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4987 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4988 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4989 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4990 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4991 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4992 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4993 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4994 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4995
4996 /* Guest bits. */
4997 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4998 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4999 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5000 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5001 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5002 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5003 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5004 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5005
5006 /* Host bits. */
5007 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5008 Log4(("Host CR0 %#RHr\n", uHCReg));
5009 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5010 Log4(("Host CR3 %#RHr\n", uHCReg));
5011 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5012 Log4(("Host CR4 %#RHr\n", uHCReg));
5013
5014 RTGDTR HostGdtr;
5015 PCX86DESCHC pDesc;
5016 ASMGetGDTR(&HostGdtr);
5017 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5018 Log4(("Host CS %#08x\n", u32Val));
5019 if (u32Val < HostGdtr.cbGdt)
5020 {
5021 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5022 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5023 }
5024
5025 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5026 Log4(("Host DS %#08x\n", u32Val));
5027 if (u32Val < HostGdtr.cbGdt)
5028 {
5029 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5030 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5031 }
5032
5033 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5034 Log4(("Host ES %#08x\n", u32Val));
5035 if (u32Val < HostGdtr.cbGdt)
5036 {
5037 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5038 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5039 }
5040
5041 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5042 Log4(("Host FS %#08x\n", u32Val));
5043 if (u32Val < HostGdtr.cbGdt)
5044 {
5045 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5046 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5047 }
5048
5049 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5050 Log4(("Host GS %#08x\n", u32Val));
5051 if (u32Val < HostGdtr.cbGdt)
5052 {
5053 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5054 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5055 }
5056
5057 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5058 Log4(("Host SS %#08x\n", u32Val));
5059 if (u32Val < HostGdtr.cbGdt)
5060 {
5061 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5062 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5063 }
5064
5065 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5066 Log4(("Host TR %#08x\n", u32Val));
5067 if (u32Val < HostGdtr.cbGdt)
5068 {
5069 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5070 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5071 }
5072
5073 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5074 Log4(("Host TR Base %#RHv\n", uHCReg));
5075 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5076 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5077 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5078 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5079 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5080 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5081 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5082 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5083 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5084 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5085 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5086 Log4(("Host RSP %#RHv\n", uHCReg));
5087 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5088 Log4(("Host RIP %#RHv\n", uHCReg));
5089# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5090 if (HMVMX_IS_64BIT_HOST_MODE())
5091 {
5092 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5093 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5094 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5095 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5096 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5097 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5098 }
5099# endif
5100#endif /* VBOX_STRICT */
5101 break;
5102 }
5103
5104 default:
5105 /* Impossible */
5106 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5107 break;
5108 }
5109 NOREF(pVM); NOREF(pCtx);
5110}
5111
5112
5113#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5114#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5115# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5116#endif
5117#ifdef VBOX_STRICT
5118static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5119{
5120 switch (idxField)
5121 {
5122 case VMX_VMCS_GUEST_RIP:
5123 case VMX_VMCS_GUEST_RSP:
5124 case VMX_VMCS_GUEST_SYSENTER_EIP:
5125 case VMX_VMCS_GUEST_SYSENTER_ESP:
5126 case VMX_VMCS_GUEST_GDTR_BASE:
5127 case VMX_VMCS_GUEST_IDTR_BASE:
5128 case VMX_VMCS_GUEST_CS_BASE:
5129 case VMX_VMCS_GUEST_DS_BASE:
5130 case VMX_VMCS_GUEST_ES_BASE:
5131 case VMX_VMCS_GUEST_FS_BASE:
5132 case VMX_VMCS_GUEST_GS_BASE:
5133 case VMX_VMCS_GUEST_SS_BASE:
5134 case VMX_VMCS_GUEST_LDTR_BASE:
5135 case VMX_VMCS_GUEST_TR_BASE:
5136 case VMX_VMCS_GUEST_CR3:
5137 return true;
5138 }
5139 return false;
5140}
5141
5142static bool hmR0VmxIsValidReadField(uint32_t idxField)
5143{
5144 switch (idxField)
5145 {
5146 /* Read-only fields. */
5147 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5148 return true;
5149 }
5150 /* Remaining readable fields should also be writable. */
5151 return hmR0VmxIsValidWriteField(idxField);
5152}
5153#endif /* VBOX_STRICT */
5154
5155
5156/**
5157 * Executes the specified handler in 64-bit mode.
5158 *
5159 * @returns VBox status code.
5160 * @param pVM Pointer to the VM.
5161 * @param pVCpu Pointer to the VMCPU.
5162 * @param pCtx Pointer to the guest CPU context.
5163 * @param enmOp The operation to perform.
5164 * @param cbParam Number of parameters.
5165 * @param paParam Array of 32-bit parameters.
5166 */
5167VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5168 uint32_t *paParam)
5169{
5170 int rc, rc2;
5171 PHMGLOBALCPUINFO pCpu;
5172 RTHCPHYS HCPhysCpuPage;
5173 RTCCUINTREG uOldEflags;
5174
5175 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5176 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5177 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5178 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5179
5180#ifdef VBOX_STRICT
5181 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5182 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5183
5184 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5185 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5186#endif
5187
5188 /* Disable interrupts. */
5189 uOldEflags = ASMIntDisableFlags();
5190
5191#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5192 RTCPUID idHostCpu = RTMpCpuId();
5193 CPUMR0SetLApic(pVCpu, idHostCpu);
5194#endif
5195
5196 pCpu = HMR0GetCurrentCpu();
5197 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5198
5199 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5200 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5201
5202 /* Leave VMX Root Mode. */
5203 VMXDisable();
5204
5205 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5206
5207 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5208 CPUMSetHyperEIP(pVCpu, enmOp);
5209 for (int i = (int)cbParam - 1; i >= 0; i--)
5210 CPUMPushHyper(pVCpu, paParam[i]);
5211
5212 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5213
5214 /* Call the switcher. */
5215 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5216 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5217
5218 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5219 /* Make sure the VMX instructions don't cause #UD faults. */
5220 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5221
5222 /* Re-enter VMX Root Mode */
5223 rc2 = VMXEnable(HCPhysCpuPage);
5224 if (RT_FAILURE(rc2))
5225 {
5226 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5227 ASMSetFlags(uOldEflags);
5228 return rc2;
5229 }
5230
5231 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5232 AssertRC(rc2);
5233 Assert(!(ASMGetFlags() & X86_EFL_IF));
5234 ASMSetFlags(uOldEflags);
5235 return rc;
5236}
5237
5238
5239/**
5240 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5241 * supporting 64-bit guests.
5242 *
5243 * @returns VBox status code.
5244 * @param fResume Whether to VMLAUNCH or VMRESUME.
5245 * @param pCtx Pointer to the guest-CPU context.
5246 * @param pCache Pointer to the VMCS cache.
5247 * @param pVM Pointer to the VM.
5248 * @param pVCpu Pointer to the VMCPU.
5249 */
5250DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5251{
5252 uint32_t aParam[6];
5253 PHMGLOBALCPUINFO pCpu = NULL;
5254 RTHCPHYS HCPhysCpuPage = 0;
5255 int rc = VERR_INTERNAL_ERROR_5;
5256
5257 pCpu = HMR0GetCurrentCpu();
5258 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5259
5260#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5261 pCache->uPos = 1;
5262 pCache->interPD = PGMGetInterPaeCR3(pVM);
5263 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5264#endif
5265
5266#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5267 pCache->TestIn.HCPhysCpuPage = 0;
5268 pCache->TestIn.HCPhysVmcs = 0;
5269 pCache->TestIn.pCache = 0;
5270 pCache->TestOut.HCPhysVmcs = 0;
5271 pCache->TestOut.pCache = 0;
5272 pCache->TestOut.pCtx = 0;
5273 pCache->TestOut.eflags = 0;
5274#endif
5275
5276 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5277 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5278 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5279 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5280 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5281 aParam[5] = 0;
5282
5283#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5284 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5285 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5286#endif
5287 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5288
5289#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5290 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5291 Assert(pCtx->dr[4] == 10);
5292 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5293#endif
5294
5295#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5296 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5297 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5298 pVCpu->hm.s.vmx.HCPhysVmcs));
5299 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5300 pCache->TestOut.HCPhysVmcs));
5301 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5302 pCache->TestOut.pCache));
5303 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5304 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5305 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5306 pCache->TestOut.pCtx));
5307 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5308#endif
5309 return rc;
5310}
5311
5312
5313/**
5314 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5315 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5316 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5317 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5318 *
5319 * @returns VBox status code.
5320 * @param pVM Pointer to the VM.
5321 * @param pVCpu Pointer to the VMCPU.
5322 */
5323static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5324{
5325#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5326{ \
5327 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5328 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5329 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5330 ++cReadFields; \
5331}
5332
5333 AssertPtr(pVM);
5334 AssertPtr(pVCpu);
5335 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5336 uint32_t cReadFields = 0;
5337
5338 /*
5339 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5340 * and serve to indicate exceptions to the rules.
5341 */
5342
5343 /* Guest-natural selector base fields. */
5344#if 0
5345 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5346 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5347 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5348#endif
5349 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5350 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5351 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5352 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5353 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5354 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5355 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5357 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5361#if 0
5362 /* Unused natural width guest-state fields. */
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5365#endif
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5368
5369 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5370#if 0
5371 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5380#endif
5381
5382 /* Natural width guest-state fields. */
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5384#if 0
5385 /* Currently unused field. */
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5387#endif
5388
5389 if (pVM->hm.s.fNestedPaging)
5390 {
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5392 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5393 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5394 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5395 }
5396 else
5397 {
5398 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5399 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5400 }
5401
5402#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5403 return VINF_SUCCESS;
5404}
5405
5406
5407/**
5408 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5409 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5410 * darwin, running 64-bit guests).
5411 *
5412 * @returns VBox status code.
5413 * @param pVCpu Pointer to the VMCPU.
5414 * @param idxField The VMCS field encoding.
5415 * @param u64Val 16, 32 or 64-bit value.
5416 */
5417VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5418{
5419 int rc;
5420 switch (idxField)
5421 {
5422 /*
5423 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5424 */
5425 /* 64-bit Control fields. */
5426 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5427 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5428 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5429 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5430 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5431 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5432 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5433 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5434 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5435 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5436 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5437 case VMX_VMCS64_CTRL_EPTP_FULL:
5438 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5439 /* 64-bit Guest-state fields. */
5440 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5441 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5442 case VMX_VMCS64_GUEST_PAT_FULL:
5443 case VMX_VMCS64_GUEST_EFER_FULL:
5444 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5445 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5446 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5447 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5448 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5449 /* 64-bit Host-state fields. */
5450 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5451 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5452 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5453 {
5454 rc = VMXWriteVmcs32(idxField, u64Val);
5455 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5456 break;
5457 }
5458
5459 /*
5460 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5461 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5462 */
5463 /* Natural-width Guest-state fields. */
5464 case VMX_VMCS_GUEST_CR3:
5465 case VMX_VMCS_GUEST_ES_BASE:
5466 case VMX_VMCS_GUEST_CS_BASE:
5467 case VMX_VMCS_GUEST_SS_BASE:
5468 case VMX_VMCS_GUEST_DS_BASE:
5469 case VMX_VMCS_GUEST_FS_BASE:
5470 case VMX_VMCS_GUEST_GS_BASE:
5471 case VMX_VMCS_GUEST_LDTR_BASE:
5472 case VMX_VMCS_GUEST_TR_BASE:
5473 case VMX_VMCS_GUEST_GDTR_BASE:
5474 case VMX_VMCS_GUEST_IDTR_BASE:
5475 case VMX_VMCS_GUEST_RSP:
5476 case VMX_VMCS_GUEST_RIP:
5477 case VMX_VMCS_GUEST_SYSENTER_ESP:
5478 case VMX_VMCS_GUEST_SYSENTER_EIP:
5479 {
5480 if (!(u64Val >> 32))
5481 {
5482 /* If this field is 64-bit, VT-x will zero out the top bits. */
5483 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5484 }
5485 else
5486 {
5487 /* Assert that only the 32->64 switcher case should ever come here. */
5488 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5489 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5490 }
5491 break;
5492 }
5493
5494 default:
5495 {
5496 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5497 rc = VERR_INVALID_PARAMETER;
5498 break;
5499 }
5500 }
5501 AssertRCReturn(rc, rc);
5502 return rc;
5503}
5504
5505
5506/**
5507 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5508 * hosts (except darwin) for 64-bit guests.
5509 *
5510 * @param pVCpu Pointer to the VMCPU.
5511 * @param idxField The VMCS field encoding.
5512 * @param u64Val 16, 32 or 64-bit value.
5513 */
5514VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5515{
5516 AssertPtr(pVCpu);
5517 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5518
5519 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5520 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5521
5522 /* Make sure there are no duplicates. */
5523 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5524 {
5525 if (pCache->Write.aField[i] == idxField)
5526 {
5527 pCache->Write.aFieldVal[i] = u64Val;
5528 return VINF_SUCCESS;
5529 }
5530 }
5531
5532 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5533 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5534 pCache->Write.cValidEntries++;
5535 return VINF_SUCCESS;
5536}
5537
5538/* Enable later when the assembly code uses these as callbacks. */
5539#if 0
5540/*
5541 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5542 *
5543 * @param pVCpu Pointer to the VMCPU.
5544 * @param pCache Pointer to the VMCS cache.
5545 *
5546 * @remarks No-long-jump zone!!!
5547 */
5548VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5549{
5550 AssertPtr(pCache);
5551 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5552 {
5553 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5554 AssertRC(rc);
5555 }
5556 pCache->Write.cValidEntries = 0;
5557}
5558
5559
5560/**
5561 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5562 *
5563 * @param pVCpu Pointer to the VMCPU.
5564 * @param pCache Pointer to the VMCS cache.
5565 *
5566 * @remarks No-long-jump zone!!!
5567 */
5568VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5569{
5570 AssertPtr(pCache);
5571 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5572 {
5573 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5574 AssertRC(rc);
5575 }
5576}
5577#endif
5578#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5579
5580
5581/**
5582 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5583 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5584 * timer.
5585 *
5586 * @returns VBox status code.
5587 * @param pVCpu Pointer to the VMCPU.
5588 *
5589 * @remarks No-long-jump zone!!!
5590 */
5591static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5592{
5593 int rc = VERR_INTERNAL_ERROR_5;
5594 bool fOffsettedTsc = false;
5595 bool fParavirtTsc = false;
5596 PVM pVM = pVCpu->CTX_SUFF(pVM);
5597 if (pVM->hm.s.vmx.fUsePreemptTimer)
5598 {
5599 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &fParavirtTsc,
5600 &pVCpu->hm.s.vmx.u64TSCOffset);
5601
5602 /* Make sure the returned values have sane upper and lower boundaries. */
5603 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5604 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5605 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5606 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5607
5608 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5609 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5610 }
5611 else
5612 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5613
5614 if (fParavirtTsc)
5615 {
5616 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5617 AssertRC(rc);
5618 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5619 }
5620
5621 if (fOffsettedTsc)
5622 {
5623 uint64_t u64CurTSC = ASMReadTSC();
5624 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5625 {
5626 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5627 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5628
5629 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5630 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5631 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5632 }
5633 else
5634 {
5635 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5636 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5637 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5638 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5639 }
5640 }
5641 else
5642 {
5643 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5644 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5645 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5646 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5647 }
5648}
5649
5650
5651/**
5652 * Determines if an exception is a contributory exception. Contributory
5653 * exceptions are ones which can cause double-faults. Page-fault is
5654 * intentionally not included here as it's a conditional contributory exception.
5655 *
5656 * @returns true if the exception is contributory, false otherwise.
5657 * @param uVector The exception vector.
5658 */
5659DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5660{
5661 switch (uVector)
5662 {
5663 case X86_XCPT_GP:
5664 case X86_XCPT_SS:
5665 case X86_XCPT_NP:
5666 case X86_XCPT_TS:
5667 case X86_XCPT_DE:
5668 return true;
5669 default:
5670 break;
5671 }
5672 return false;
5673}
5674
5675
5676/**
5677 * Sets an event as a pending event to be injected into the guest.
5678 *
5679 * @param pVCpu Pointer to the VMCPU.
5680 * @param u32IntInfo The VM-entry interruption-information field.
5681 * @param cbInstr The VM-entry instruction length in bytes (for software
5682 * interrupts, exceptions and privileged software
5683 * exceptions).
5684 * @param u32ErrCode The VM-entry exception error code.
5685 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5686 * page-fault.
5687 *
5688 * @remarks Statistics counter assumes this is a guest event being injected or
5689 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5690 * always incremented.
5691 */
5692DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5693 RTGCUINTPTR GCPtrFaultAddress)
5694{
5695 Assert(!pVCpu->hm.s.Event.fPending);
5696 pVCpu->hm.s.Event.fPending = true;
5697 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5698 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5699 pVCpu->hm.s.Event.cbInstr = cbInstr;
5700 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5701
5702 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5703}
5704
5705
5706/**
5707 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5708 *
5709 * @param pVCpu Pointer to the VMCPU.
5710 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5711 * out-of-sync. Make sure to update the required fields
5712 * before using them.
5713 */
5714DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5715{
5716 NOREF(pMixedCtx);
5717 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5718 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5719 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5720 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5721}
5722
5723
5724/**
5725 * Handle a condition that occurred while delivering an event through the guest
5726 * IDT.
5727 *
5728 * @returns VBox status code (informational error codes included).
5729 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5730 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5731 * continue execution of the guest which will delivery the #DF.
5732 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5733 *
5734 * @param pVCpu Pointer to the VMCPU.
5735 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5736 * out-of-sync. Make sure to update the required fields
5737 * before using them.
5738 * @param pVmxTransient Pointer to the VMX transient structure.
5739 *
5740 * @remarks No-long-jump zone!!!
5741 */
5742static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5743{
5744 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5745
5746 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5747 AssertRCReturn(rc, rc);
5748 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5749 {
5750 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5751 AssertRCReturn(rc, rc);
5752
5753 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5754 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5755
5756 typedef enum
5757 {
5758 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5759 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5760 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5761 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5762 } VMXREFLECTXCPT;
5763
5764 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5765 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5766 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5767 {
5768 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5769 {
5770 enmReflect = VMXREFLECTXCPT_XCPT;
5771#ifdef VBOX_STRICT
5772 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5773 && uExitVector == X86_XCPT_PF)
5774 {
5775 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5776 }
5777#endif
5778 if ( uExitVector == X86_XCPT_PF
5779 && uIdtVector == X86_XCPT_PF)
5780 {
5781 pVmxTransient->fVectoringDoublePF = true;
5782 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5783 }
5784 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5785 && hmR0VmxIsContributoryXcpt(uExitVector)
5786 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5787 || uIdtVector == X86_XCPT_PF))
5788 {
5789 enmReflect = VMXREFLECTXCPT_DF;
5790 }
5791 else if (uIdtVector == X86_XCPT_DF)
5792 enmReflect = VMXREFLECTXCPT_TF;
5793 }
5794 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5795 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5796 {
5797 /*
5798 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5799 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5800 */
5801 enmReflect = VMXREFLECTXCPT_XCPT;
5802
5803 if (uExitVector == X86_XCPT_PF)
5804 {
5805 pVmxTransient->fVectoringPF = true;
5806 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5807 }
5808 }
5809 }
5810 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5811 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5812 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5813 {
5814 /*
5815 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5816 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5817 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5818 */
5819 enmReflect = VMXREFLECTXCPT_XCPT;
5820 }
5821
5822 /*
5823 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5824 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5825 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5826 *
5827 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5828 */
5829 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5830 && enmReflect == VMXREFLECTXCPT_XCPT
5831 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5832 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5833 {
5834 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5835 }
5836
5837 switch (enmReflect)
5838 {
5839 case VMXREFLECTXCPT_XCPT:
5840 {
5841 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5842 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5843 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5844
5845 uint32_t u32ErrCode = 0;
5846 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5847 {
5848 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5849 AssertRCReturn(rc, rc);
5850 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5851 }
5852
5853 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5854 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5855 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5856 rc = VINF_SUCCESS;
5857 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5858 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5859
5860 break;
5861 }
5862
5863 case VMXREFLECTXCPT_DF:
5864 {
5865 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5866 rc = VINF_HM_DOUBLE_FAULT;
5867 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5868 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5869
5870 break;
5871 }
5872
5873 case VMXREFLECTXCPT_TF:
5874 {
5875 rc = VINF_EM_RESET;
5876 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5877 uExitVector));
5878 break;
5879 }
5880
5881 default:
5882 Assert(rc == VINF_SUCCESS);
5883 break;
5884 }
5885 }
5886 else if ( VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5887 && uExitVector != X86_XCPT_DF
5888 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5889 {
5890 /*
5891 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5892 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5893 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5894 */
5895 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5896 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5897 }
5898
5899 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5900 return rc;
5901}
5902
5903
5904/**
5905 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5906 *
5907 * @returns VBox status code.
5908 * @param pVCpu Pointer to the VMCPU.
5909 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5910 * out-of-sync. Make sure to update the required fields
5911 * before using them.
5912 *
5913 * @remarks No-long-jump zone!!!
5914 */
5915static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5916{
5917 NOREF(pMixedCtx);
5918
5919 /*
5920 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5921 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5922 */
5923 VMMRZCallRing3Disable(pVCpu);
5924 HM_DISABLE_PREEMPT_IF_NEEDED();
5925
5926 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5927 {
5928 uint32_t uVal = 0;
5929 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5930 AssertRCReturn(rc, rc);
5931
5932 uint32_t uShadow = 0;
5933 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5934 AssertRCReturn(rc, rc);
5935
5936 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5937 CPUMSetGuestCR0(pVCpu, uVal);
5938 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5939 }
5940
5941 HM_RESTORE_PREEMPT_IF_NEEDED();
5942 VMMRZCallRing3Enable(pVCpu);
5943 return VINF_SUCCESS;
5944}
5945
5946
5947/**
5948 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5949 *
5950 * @returns VBox status code.
5951 * @param pVCpu Pointer to the VMCPU.
5952 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5953 * out-of-sync. Make sure to update the required fields
5954 * before using them.
5955 *
5956 * @remarks No-long-jump zone!!!
5957 */
5958static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5959{
5960 NOREF(pMixedCtx);
5961
5962 int rc = VINF_SUCCESS;
5963 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5964 {
5965 uint32_t uVal = 0;
5966 uint32_t uShadow = 0;
5967 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5968 AssertRCReturn(rc, rc);
5969 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5970 AssertRCReturn(rc, rc);
5971
5972 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5973 CPUMSetGuestCR4(pVCpu, uVal);
5974 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5975 }
5976 return rc;
5977}
5978
5979
5980/**
5981 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5982 *
5983 * @returns VBox status code.
5984 * @param pVCpu Pointer to the VMCPU.
5985 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5986 * out-of-sync. Make sure to update the required fields
5987 * before using them.
5988 *
5989 * @remarks No-long-jump zone!!!
5990 */
5991static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5992{
5993 int rc = VINF_SUCCESS;
5994 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5995 {
5996 uint64_t u64Val = 0;
5997 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5998 AssertRCReturn(rc, rc);
5999
6000 pMixedCtx->rip = u64Val;
6001 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6002 }
6003 return rc;
6004}
6005
6006
6007/**
6008 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6009 *
6010 * @returns VBox status code.
6011 * @param pVCpu Pointer to the VMCPU.
6012 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6013 * out-of-sync. Make sure to update the required fields
6014 * before using them.
6015 *
6016 * @remarks No-long-jump zone!!!
6017 */
6018static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6019{
6020 int rc = VINF_SUCCESS;
6021 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6022 {
6023 uint64_t u64Val = 0;
6024 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6025 AssertRCReturn(rc, rc);
6026
6027 pMixedCtx->rsp = u64Val;
6028 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6029 }
6030 return rc;
6031}
6032
6033
6034/**
6035 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6036 *
6037 * @returns VBox status code.
6038 * @param pVCpu Pointer to the VMCPU.
6039 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6040 * out-of-sync. Make sure to update the required fields
6041 * before using them.
6042 *
6043 * @remarks No-long-jump zone!!!
6044 */
6045static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6046{
6047 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6048 {
6049 uint32_t uVal = 0;
6050 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6051 AssertRCReturn(rc, rc);
6052
6053 pMixedCtx->eflags.u32 = uVal;
6054 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6055 {
6056 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6057 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6058
6059 pMixedCtx->eflags.Bits.u1VM = 0;
6060 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6061 }
6062
6063 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6064 }
6065 return VINF_SUCCESS;
6066}
6067
6068
6069/**
6070 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6071 * guest-CPU context.
6072 */
6073DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6074{
6075 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6076 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6077 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6078 return rc;
6079}
6080
6081
6082/**
6083 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6084 * from the guest-state area in the VMCS.
6085 *
6086 * @param pVCpu Pointer to the VMCPU.
6087 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6088 * out-of-sync. Make sure to update the required fields
6089 * before using them.
6090 *
6091 * @remarks No-long-jump zone!!!
6092 */
6093static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6094{
6095 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6096 {
6097 uint32_t uIntrState = 0;
6098 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6099 AssertRC(rc);
6100
6101 if (!uIntrState)
6102 {
6103 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6104 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6105
6106 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6107 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6108 }
6109 else
6110 {
6111 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6112 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6113 {
6114 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6115 AssertRC(rc);
6116 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6117 AssertRC(rc);
6118
6119 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6120 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6121 }
6122 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6123 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6124
6125 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6126 {
6127 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6128 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6129 }
6130 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6131 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6132 }
6133
6134 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6135 }
6136}
6137
6138
6139/**
6140 * Saves the guest's activity state.
6141 *
6142 * @returns VBox status code.
6143 * @param pVCpu Pointer to the VMCPU.
6144 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6145 * out-of-sync. Make sure to update the required fields
6146 * before using them.
6147 *
6148 * @remarks No-long-jump zone!!!
6149 */
6150static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6151{
6152 NOREF(pMixedCtx);
6153 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6154 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6155 return VINF_SUCCESS;
6156}
6157
6158
6159/**
6160 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6161 * the current VMCS into the guest-CPU context.
6162 *
6163 * @returns VBox status code.
6164 * @param pVCpu Pointer to the VMCPU.
6165 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6166 * out-of-sync. Make sure to update the required fields
6167 * before using them.
6168 *
6169 * @remarks No-long-jump zone!!!
6170 */
6171static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6172{
6173 int rc = VINF_SUCCESS;
6174 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6175 {
6176 uint32_t u32Val = 0;
6177 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6178 pMixedCtx->SysEnter.cs = u32Val;
6179 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6180 }
6181
6182 uint64_t u64Val = 0;
6183 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6184 {
6185 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6186 pMixedCtx->SysEnter.eip = u64Val;
6187 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6188 }
6189 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6190 {
6191 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6192 pMixedCtx->SysEnter.esp = u64Val;
6193 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6194 }
6195 return rc;
6196}
6197
6198
6199/**
6200 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6201 * the CPU back into the guest-CPU context.
6202 *
6203 * @returns VBox status code.
6204 * @param pVCpu Pointer to the VMCPU.
6205 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6206 * out-of-sync. Make sure to update the required fields
6207 * before using them.
6208 *
6209 * @remarks No-long-jump zone!!!
6210 */
6211static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6212{
6213#if HC_ARCH_BITS == 64
6214 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6215 {
6216 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6217 VMMRZCallRing3Disable(pVCpu);
6218 HM_DISABLE_PREEMPT_IF_NEEDED();
6219
6220 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6221 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6222 {
6223 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6224 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6225 }
6226
6227 HM_RESTORE_PREEMPT_IF_NEEDED();
6228 VMMRZCallRing3Enable(pVCpu);
6229 }
6230 else
6231 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6232#else
6233 NOREF(pMixedCtx);
6234 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6235#endif
6236
6237 return VINF_SUCCESS;
6238}
6239
6240
6241/**
6242 * Saves the auto load/store'd guest MSRs from the current VMCS into
6243 * the guest-CPU context.
6244 *
6245 * @returns VBox status code.
6246 * @param pVCpu Pointer to the VMCPU.
6247 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6248 * out-of-sync. Make sure to update the required fields
6249 * before using them.
6250 *
6251 * @remarks No-long-jump zone!!!
6252 */
6253static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6254{
6255 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6256 return VINF_SUCCESS;
6257
6258 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6259 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6260 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6261 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6262 {
6263 switch (pMsr->u32Msr)
6264 {
6265 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6266 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6267 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6268 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6269 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6270 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6271 break;
6272
6273 default:
6274 {
6275 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6276 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6277 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6278 }
6279 }
6280 }
6281
6282 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6283 return VINF_SUCCESS;
6284}
6285
6286
6287/**
6288 * Saves the guest control registers from the current VMCS into the guest-CPU
6289 * context.
6290 *
6291 * @returns VBox status code.
6292 * @param pVCpu Pointer to the VMCPU.
6293 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6294 * out-of-sync. Make sure to update the required fields
6295 * before using them.
6296 *
6297 * @remarks No-long-jump zone!!!
6298 */
6299static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6300{
6301 /* Guest CR0. Guest FPU. */
6302 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6303 AssertRCReturn(rc, rc);
6304
6305 /* Guest CR4. */
6306 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6307 AssertRCReturn(rc, rc);
6308
6309 /* Guest CR2 - updated always during the world-switch or in #PF. */
6310 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6311 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6312 {
6313 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6314 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6315
6316 PVM pVM = pVCpu->CTX_SUFF(pVM);
6317 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6318 || ( pVM->hm.s.fNestedPaging
6319 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6320 {
6321 uint64_t u64Val = 0;
6322 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6323 if (pMixedCtx->cr3 != u64Val)
6324 {
6325 CPUMSetGuestCR3(pVCpu, u64Val);
6326 if (VMMRZCallRing3IsEnabled(pVCpu))
6327 {
6328 PGMUpdateCR3(pVCpu, u64Val);
6329 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6330 }
6331 else
6332 {
6333 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6334 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6335 }
6336 }
6337
6338 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6339 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6340 {
6341 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6342 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6343 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6344 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6345
6346 if (VMMRZCallRing3IsEnabled(pVCpu))
6347 {
6348 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6349 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6350 }
6351 else
6352 {
6353 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6354 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6355 }
6356 }
6357 }
6358
6359 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6360 }
6361
6362 /*
6363 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6364 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6365 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6366 *
6367 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6368 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6369 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6370 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6371 *
6372 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6373 */
6374 if (VMMRZCallRing3IsEnabled(pVCpu))
6375 {
6376 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6377 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6378
6379 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6380 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6381
6382 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6383 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6384 }
6385
6386 return rc;
6387}
6388
6389
6390/**
6391 * Reads a guest segment register from the current VMCS into the guest-CPU
6392 * context.
6393 *
6394 * @returns VBox status code.
6395 * @param pVCpu Pointer to the VMCPU.
6396 * @param idxSel Index of the selector in the VMCS.
6397 * @param idxLimit Index of the segment limit in the VMCS.
6398 * @param idxBase Index of the segment base in the VMCS.
6399 * @param idxAccess Index of the access rights of the segment in the VMCS.
6400 * @param pSelReg Pointer to the segment selector.
6401 *
6402 * @remarks No-long-jump zone!!!
6403 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6404 * macro as that takes care of whether to read from the VMCS cache or
6405 * not.
6406 */
6407DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6408 PCPUMSELREG pSelReg)
6409{
6410 NOREF(pVCpu);
6411
6412 uint32_t u32Val = 0;
6413 int rc = VMXReadVmcs32(idxSel, &u32Val);
6414 AssertRCReturn(rc, rc);
6415 pSelReg->Sel = (uint16_t)u32Val;
6416 pSelReg->ValidSel = (uint16_t)u32Val;
6417 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6418
6419 rc = VMXReadVmcs32(idxLimit, &u32Val);
6420 AssertRCReturn(rc, rc);
6421 pSelReg->u32Limit = u32Val;
6422
6423 uint64_t u64Val = 0;
6424 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6425 AssertRCReturn(rc, rc);
6426 pSelReg->u64Base = u64Val;
6427
6428 rc = VMXReadVmcs32(idxAccess, &u32Val);
6429 AssertRCReturn(rc, rc);
6430 pSelReg->Attr.u = u32Val;
6431
6432 /*
6433 * If VT-x marks the segment as unusable, most other bits remain undefined:
6434 * - For CS the L, D and G bits have meaning.
6435 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6436 * - For the remaining data segments no bits are defined.
6437 *
6438 * The present bit and the unusable bit has been observed to be set at the
6439 * same time (the selector was supposed to be invalid as we started executing
6440 * a V8086 interrupt in ring-0).
6441 *
6442 * What should be important for the rest of the VBox code, is that the P bit is
6443 * cleared. Some of the other VBox code recognizes the unusable bit, but
6444 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6445 * safe side here, we'll strip off P and other bits we don't care about. If
6446 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6447 *
6448 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6449 */
6450 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6451 {
6452 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6453
6454 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6455 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6456 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6457
6458 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6459#ifdef DEBUG_bird
6460 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6461 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6462 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6463#endif
6464 }
6465 return VINF_SUCCESS;
6466}
6467
6468
6469#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6470# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6471 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6472 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6473#else
6474# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6475 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6476 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6477#endif
6478
6479
6480/**
6481 * Saves the guest segment registers from the current VMCS into the guest-CPU
6482 * context.
6483 *
6484 * @returns VBox status code.
6485 * @param pVCpu Pointer to the VMCPU.
6486 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6487 * out-of-sync. Make sure to update the required fields
6488 * before using them.
6489 *
6490 * @remarks No-long-jump zone!!!
6491 */
6492static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6493{
6494 /* Guest segment registers. */
6495 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6496 {
6497 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6498 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6499 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6500 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6501 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6502 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6503 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6504
6505 /* Restore segment attributes for real-on-v86 mode hack. */
6506 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6507 {
6508 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6509 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6510 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6511 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6512 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6513 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6514 }
6515 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6516 }
6517
6518 return VINF_SUCCESS;
6519}
6520
6521
6522/**
6523 * Saves the guest descriptor table registers and task register from the current
6524 * VMCS into the guest-CPU context.
6525 *
6526 * @returns VBox status code.
6527 * @param pVCpu Pointer to the VMCPU.
6528 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6529 * out-of-sync. Make sure to update the required fields
6530 * before using them.
6531 *
6532 * @remarks No-long-jump zone!!!
6533 */
6534static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6535{
6536 int rc = VINF_SUCCESS;
6537
6538 /* Guest LDTR. */
6539 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6540 {
6541 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6542 AssertRCReturn(rc, rc);
6543 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6544 }
6545
6546 /* Guest GDTR. */
6547 uint64_t u64Val = 0;
6548 uint32_t u32Val = 0;
6549 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6550 {
6551 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6552 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6553 pMixedCtx->gdtr.pGdt = u64Val;
6554 pMixedCtx->gdtr.cbGdt = u32Val;
6555 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6556 }
6557
6558 /* Guest IDTR. */
6559 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6560 {
6561 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6562 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6563 pMixedCtx->idtr.pIdt = u64Val;
6564 pMixedCtx->idtr.cbIdt = u32Val;
6565 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6566 }
6567
6568 /* Guest TR. */
6569 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6570 {
6571 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6572 AssertRCReturn(rc, rc);
6573
6574 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6575 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6576 {
6577 rc = VMXLOCAL_READ_SEG(TR, tr);
6578 AssertRCReturn(rc, rc);
6579 }
6580 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6581 }
6582 return rc;
6583}
6584
6585#undef VMXLOCAL_READ_SEG
6586
6587
6588/**
6589 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6590 * context.
6591 *
6592 * @returns VBox status code.
6593 * @param pVCpu Pointer to the VMCPU.
6594 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6595 * out-of-sync. Make sure to update the required fields
6596 * before using them.
6597 *
6598 * @remarks No-long-jump zone!!!
6599 */
6600static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6601{
6602 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6603 {
6604 if (!pVCpu->hm.s.fUsingHyperDR7)
6605 {
6606 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6607 uint32_t u32Val;
6608 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6609 pMixedCtx->dr[7] = u32Val;
6610 }
6611
6612 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6613 }
6614 return VINF_SUCCESS;
6615}
6616
6617
6618/**
6619 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6620 *
6621 * @returns VBox status code.
6622 * @param pVCpu Pointer to the VMCPU.
6623 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6624 * out-of-sync. Make sure to update the required fields
6625 * before using them.
6626 *
6627 * @remarks No-long-jump zone!!!
6628 */
6629static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6630{
6631 NOREF(pMixedCtx);
6632
6633 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6634 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6635 return VINF_SUCCESS;
6636}
6637
6638
6639/**
6640 * Saves the entire guest state from the currently active VMCS into the
6641 * guest-CPU context. This essentially VMREADs all guest-data.
6642 *
6643 * @returns VBox status code.
6644 * @param pVCpu Pointer to the VMCPU.
6645 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6646 * out-of-sync. Make sure to update the required fields
6647 * before using them.
6648 */
6649static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6650{
6651 Assert(pVCpu);
6652 Assert(pMixedCtx);
6653
6654 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6655 return VINF_SUCCESS;
6656
6657 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6658 again on the ring-3 callback path, there is no real need to. */
6659 if (VMMRZCallRing3IsEnabled(pVCpu))
6660 VMMR0LogFlushDisable(pVCpu);
6661 else
6662 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6663 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6664
6665 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6666 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6667
6668 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6669 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6670
6671 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6672 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6673
6674 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6675 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6676
6677 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6678 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6679
6680 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6681 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6682
6683 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6684 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6685
6686 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6687 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6688
6689 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6690 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6691
6692 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6693 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6694
6695 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6696 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6697
6698 if (VMMRZCallRing3IsEnabled(pVCpu))
6699 VMMR0LogFlushEnable(pVCpu);
6700
6701 return rc;
6702}
6703
6704
6705/**
6706 * Check per-VM and per-VCPU force flag actions that require us to go back to
6707 * ring-3 for one reason or another.
6708 *
6709 * @returns VBox status code (information status code included).
6710 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6711 * ring-3.
6712 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6713 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6714 * interrupts)
6715 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6716 * all EMTs to be in ring-3.
6717 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6718 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6719 * to the EM loop.
6720 *
6721 * @param pVM Pointer to the VM.
6722 * @param pVCpu Pointer to the VMCPU.
6723 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6724 * out-of-sync. Make sure to update the required fields
6725 * before using them.
6726 */
6727static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6728{
6729 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6730
6731 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6732 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6733 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6734 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6735 {
6736 /* We need the control registers now, make sure the guest-CPU context is updated. */
6737 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6738 AssertRCReturn(rc3, rc3);
6739
6740 /* Pending HM CR3 sync. */
6741 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6742 {
6743 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6744 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6745 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6746 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6747 }
6748
6749 /* Pending HM PAE PDPEs. */
6750 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6751 {
6752 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6753 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6754 }
6755
6756 /* Pending PGM C3 sync. */
6757 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6758 {
6759 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6760 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6761 if (rc2 != VINF_SUCCESS)
6762 {
6763 AssertRC(rc2);
6764 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6765 return rc2;
6766 }
6767 }
6768
6769 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6770 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6771 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6772 {
6773 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6774 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6775 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6776 return rc2;
6777 }
6778
6779 /* Pending VM request packets, such as hardware interrupts. */
6780 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6781 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6782 {
6783 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6784 return VINF_EM_PENDING_REQUEST;
6785 }
6786
6787 /* Pending PGM pool flushes. */
6788 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6789 {
6790 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6791 return VINF_PGM_POOL_FLUSH_PENDING;
6792 }
6793
6794 /* Pending DMA requests. */
6795 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6796 {
6797 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6798 return VINF_EM_RAW_TO_R3;
6799 }
6800 }
6801
6802 return VINF_SUCCESS;
6803}
6804
6805
6806/**
6807 * Converts any TRPM trap into a pending HM event. This is typically used when
6808 * entering from ring-3 (not longjmp returns).
6809 *
6810 * @param pVCpu Pointer to the VMCPU.
6811 */
6812static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6813{
6814 Assert(TRPMHasTrap(pVCpu));
6815 Assert(!pVCpu->hm.s.Event.fPending);
6816
6817 uint8_t uVector;
6818 TRPMEVENT enmTrpmEvent;
6819 RTGCUINT uErrCode;
6820 RTGCUINTPTR GCPtrFaultAddress;
6821 uint8_t cbInstr;
6822
6823 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6824 AssertRC(rc);
6825
6826 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6827 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6828 if (enmTrpmEvent == TRPM_TRAP)
6829 {
6830 switch (uVector)
6831 {
6832 case X86_XCPT_NMI:
6833 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6834 break;
6835
6836 case X86_XCPT_BP:
6837 case X86_XCPT_OF:
6838 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6839 break;
6840
6841 case X86_XCPT_PF:
6842 case X86_XCPT_DF:
6843 case X86_XCPT_TS:
6844 case X86_XCPT_NP:
6845 case X86_XCPT_SS:
6846 case X86_XCPT_GP:
6847 case X86_XCPT_AC:
6848 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6849 /* no break! */
6850 default:
6851 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6852 break;
6853 }
6854 }
6855 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6856 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6857 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6858 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6859 else
6860 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6861
6862 rc = TRPMResetTrap(pVCpu);
6863 AssertRC(rc);
6864 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6865 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6866
6867 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6868 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6869}
6870
6871
6872/**
6873 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6874 * VT-x to execute any instruction.
6875 *
6876 * @param pvCpu Pointer to the VMCPU.
6877 */
6878static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6879{
6880 Assert(pVCpu->hm.s.Event.fPending);
6881
6882 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6883 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6884 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6885 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6886
6887 /* If a trap was already pending, we did something wrong! */
6888 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6889
6890 TRPMEVENT enmTrapType;
6891 switch (uVectorType)
6892 {
6893 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6894 enmTrapType = TRPM_HARDWARE_INT;
6895 break;
6896
6897 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6898 enmTrapType = TRPM_SOFTWARE_INT;
6899 break;
6900
6901 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6902 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6903 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6904 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6905 enmTrapType = TRPM_TRAP;
6906 break;
6907
6908 default:
6909 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6910 enmTrapType = TRPM_32BIT_HACK;
6911 break;
6912 }
6913
6914 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6915
6916 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6917 AssertRC(rc);
6918
6919 if (fErrorCodeValid)
6920 TRPMSetErrorCode(pVCpu, uErrorCode);
6921
6922 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6923 && uVector == X86_XCPT_PF)
6924 {
6925 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6926 }
6927 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6928 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6929 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6930 {
6931 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6932 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6933 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6934 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6935 }
6936 pVCpu->hm.s.Event.fPending = false;
6937}
6938
6939
6940/**
6941 * Does the necessary state syncing before returning to ring-3 for any reason
6942 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6943 *
6944 * @returns VBox status code.
6945 * @param pVM Pointer to the VM.
6946 * @param pVCpu Pointer to the VMCPU.
6947 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6948 * be out-of-sync. Make sure to update the required
6949 * fields before using them.
6950 * @param fSaveGuestState Whether to save the guest state or not.
6951 *
6952 * @remarks No-long-jmp zone!!!
6953 */
6954static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6955{
6956 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6957 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6958
6959 RTCPUID idCpu = RTMpCpuId();
6960 Log4Func(("HostCpuId=%u\n", idCpu));
6961
6962 /*
6963 * !!! IMPORTANT !!!
6964 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6965 */
6966
6967 /* Save the guest state if necessary. */
6968 if ( fSaveGuestState
6969 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6970 {
6971 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6972 AssertRCReturn(rc, rc);
6973 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6974 }
6975
6976 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6977 if (CPUMIsGuestFPUStateActive(pVCpu))
6978 {
6979 /* We shouldn't reload CR0 without saving it first. */
6980 if (!fSaveGuestState)
6981 {
6982 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6983 AssertRCReturn(rc, rc);
6984 }
6985 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6986 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6987 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6988 }
6989
6990 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6991#ifdef VBOX_STRICT
6992 if (CPUMIsHyperDebugStateActive(pVCpu))
6993 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6994#endif
6995 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6996 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6997 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6998 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6999
7000#if HC_ARCH_BITS == 64
7001 /* Restore host-state bits that VT-x only restores partially. */
7002 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7003 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7004 {
7005 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7006 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7007 }
7008 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7009#endif
7010
7011#if HC_ARCH_BITS == 64
7012 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7013 if ( pVM->hm.s.fAllow64BitGuests
7014 && pVCpu->hm.s.vmx.fLazyMsrs)
7015 {
7016 /* We shouldn't reload the guest MSRs without saving it first. */
7017 if (!fSaveGuestState)
7018 {
7019 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7020 AssertRCReturn(rc, rc);
7021 }
7022 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7023 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7024 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7025 }
7026#endif
7027
7028 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7029 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7030
7031 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7032 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7033 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7034 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7035 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7036 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7037 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7038 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7039
7040 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7041
7042 /** @todo This partially defeats the purpose of having preemption hooks.
7043 * The problem is, deregistering the hooks should be moved to a place that
7044 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7045 * context.
7046 */
7047 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7048 {
7049 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7050 AssertRCReturn(rc, rc);
7051
7052 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7053 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7054 }
7055 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7056 NOREF(idCpu);
7057
7058 return VINF_SUCCESS;
7059}
7060
7061
7062/**
7063 * Leaves the VT-x session.
7064 *
7065 * @returns VBox status code.
7066 * @param pVM Pointer to the VM.
7067 * @param pVCpu Pointer to the VMCPU.
7068 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7069 * out-of-sync. Make sure to update the required fields
7070 * before using them.
7071 *
7072 * @remarks No-long-jmp zone!!!
7073 */
7074DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7075{
7076 HM_DISABLE_PREEMPT_IF_NEEDED();
7077 HMVMX_ASSERT_CPU_SAFE();
7078 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7079 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7080
7081 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7082 and done this from the VMXR0ThreadCtxCallback(). */
7083 if (!pVCpu->hm.s.fLeaveDone)
7084 {
7085 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7086 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7087 pVCpu->hm.s.fLeaveDone = true;
7088 }
7089 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7090
7091 /*
7092 * !!! IMPORTANT !!!
7093 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7094 */
7095
7096 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7097 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7098 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7099 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7100 VMMR0ThreadCtxHooksDeregister(pVCpu);
7101
7102 /* Leave HM context. This takes care of local init (term). */
7103 int rc = HMR0LeaveCpu(pVCpu);
7104
7105 HM_RESTORE_PREEMPT_IF_NEEDED();
7106
7107 return rc;
7108}
7109
7110
7111/**
7112 * Does the necessary state syncing before doing a longjmp to ring-3.
7113 *
7114 * @returns VBox status code.
7115 * @param pVM Pointer to the VM.
7116 * @param pVCpu Pointer to the VMCPU.
7117 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7118 * out-of-sync. Make sure to update the required fields
7119 * before using them.
7120 *
7121 * @remarks No-long-jmp zone!!!
7122 */
7123DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7124{
7125 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7126}
7127
7128
7129/**
7130 * Take necessary actions before going back to ring-3.
7131 *
7132 * An action requires us to go back to ring-3. This function does the necessary
7133 * steps before we can safely return to ring-3. This is not the same as longjmps
7134 * to ring-3, this is voluntary and prepares the guest so it may continue
7135 * executing outside HM (recompiler/IEM).
7136 *
7137 * @returns VBox status code.
7138 * @param pVM Pointer to the VM.
7139 * @param pVCpu Pointer to the VMCPU.
7140 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7141 * out-of-sync. Make sure to update the required fields
7142 * before using them.
7143 * @param rcExit The reason for exiting to ring-3. Can be
7144 * VINF_VMM_UNKNOWN_RING3_CALL.
7145 */
7146static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7147{
7148 Assert(pVM);
7149 Assert(pVCpu);
7150 Assert(pMixedCtx);
7151 HMVMX_ASSERT_PREEMPT_SAFE();
7152
7153 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7154 {
7155 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7156 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7157 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7158 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7159 }
7160
7161 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7162 VMMRZCallRing3Disable(pVCpu);
7163 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7164
7165 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7166 if (pVCpu->hm.s.Event.fPending)
7167 {
7168 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7169 Assert(!pVCpu->hm.s.Event.fPending);
7170 }
7171
7172 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7173 and if we're injecting an event we should have a TRPM trap pending. */
7174 Assert(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu));
7175 Assert(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu));
7176
7177 /* Save guest state and restore host state bits. */
7178 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7179 AssertRCReturn(rc, rc);
7180 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7181 /* Thread-context hooks are unregistered at this point!!! */
7182
7183 /* Sync recompiler state. */
7184 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7185 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7186 | CPUM_CHANGED_LDTR
7187 | CPUM_CHANGED_GDTR
7188 | CPUM_CHANGED_IDTR
7189 | CPUM_CHANGED_TR
7190 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7191 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7192 if ( pVM->hm.s.fNestedPaging
7193 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7194 {
7195 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7196 }
7197
7198 Assert(!pVCpu->hm.s.fClearTrapFlag);
7199
7200 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7201 if (rcExit != VINF_EM_RAW_INTERRUPT)
7202 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7203
7204 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7205
7206 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7207 VMMRZCallRing3RemoveNotification(pVCpu);
7208 VMMRZCallRing3Enable(pVCpu);
7209
7210 return rc;
7211}
7212
7213
7214/**
7215 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7216 * longjump to ring-3 and possibly get preempted.
7217 *
7218 * @returns VBox status code.
7219 * @param pVCpu Pointer to the VMCPU.
7220 * @param enmOperation The operation causing the ring-3 longjump.
7221 * @param pvUser Opaque pointer to the guest-CPU context. The data
7222 * may be out-of-sync. Make sure to update the required
7223 * fields before using them.
7224 */
7225DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7226{
7227 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7228 {
7229 /*
7230 * !!! IMPORTANT !!!
7231 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7232 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7233 */
7234 VMMRZCallRing3RemoveNotification(pVCpu);
7235 VMMRZCallRing3Disable(pVCpu);
7236 HM_DISABLE_PREEMPT_IF_NEEDED();
7237
7238 PVM pVM = pVCpu->CTX_SUFF(pVM);
7239 if (CPUMIsGuestFPUStateActive(pVCpu))
7240 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7241
7242 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7243
7244#if HC_ARCH_BITS == 64
7245 /* Restore host-state bits that VT-x only restores partially. */
7246 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7247 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7248 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7249 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7250
7251 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7252 if ( pVM->hm.s.fAllow64BitGuests
7253 && pVCpu->hm.s.vmx.fLazyMsrs)
7254 {
7255 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7256 }
7257#endif
7258 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7259 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7260 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7261 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7262 {
7263 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7264 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7265 }
7266
7267 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7268 VMMR0ThreadCtxHooksDeregister(pVCpu);
7269
7270 HMR0LeaveCpu(pVCpu);
7271 HM_RESTORE_PREEMPT_IF_NEEDED();
7272 return VINF_SUCCESS;
7273 }
7274
7275 Assert(pVCpu);
7276 Assert(pvUser);
7277 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7278 HMVMX_ASSERT_PREEMPT_SAFE();
7279
7280 VMMRZCallRing3Disable(pVCpu);
7281 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7282
7283 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7284 enmOperation));
7285
7286 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7287 AssertRCReturn(rc, rc);
7288
7289 VMMRZCallRing3Enable(pVCpu);
7290 return VINF_SUCCESS;
7291}
7292
7293
7294/**
7295 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7296 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7297 *
7298 * @param pVCpu Pointer to the VMCPU.
7299 */
7300DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7301{
7302 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7303 {
7304 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7305 {
7306 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7307 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7308 AssertRC(rc);
7309 Log4(("Setup interrupt-window exiting\n"));
7310 }
7311 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7312}
7313
7314
7315/**
7316 * Clears the interrupt-window exiting control in the VMCS.
7317 *
7318 * @param pVCpu Pointer to the VMCPU.
7319 */
7320DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7321{
7322 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7323 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7324 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7325 AssertRC(rc);
7326 Log4(("Cleared interrupt-window exiting\n"));
7327}
7328
7329
7330/**
7331 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7332 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7333 *
7334 * @param pVCpu Pointer to the VMCPU.
7335 */
7336DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7337{
7338 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7339 {
7340 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7341 {
7342 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7343 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7344 AssertRC(rc);
7345 Log4(("Setup NMI-window exiting\n"));
7346 }
7347 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7348}
7349
7350
7351/**
7352 * Clears the NMI-window exiting control in the VMCS.
7353 *
7354 * @param pVCpu Pointer to the VMCPU.
7355 */
7356DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7357{
7358 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7359 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7360 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7361 AssertRC(rc);
7362 Log4(("Cleared NMI-window exiting\n"));
7363}
7364
7365
7366/**
7367 * Evaluates the event to be delivered to the guest and sets it as the pending
7368 * event.
7369 *
7370 * @param pVCpu Pointer to the VMCPU.
7371 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7372 * out-of-sync. Make sure to update the required fields
7373 * before using them.
7374 */
7375static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7376{
7377 Assert(!pVCpu->hm.s.Event.fPending);
7378
7379 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7380 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7381 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7382 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7383 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7384
7385 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7386 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7387 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7388 Assert(!TRPMHasTrap(pVCpu));
7389
7390 /*
7391 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7392 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7393 */
7394 /** @todo SMI. SMIs take priority over NMIs. */
7395 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7396 {
7397 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7398 if ( !fBlockNmi
7399 && !fBlockSti
7400 && !fBlockMovSS)
7401 {
7402 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7403 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7404 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7405
7406 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7407 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7408 }
7409 else
7410 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7411 }
7412 /*
7413 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7414 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7415 */
7416 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7417 && !pVCpu->hm.s.fSingleInstruction)
7418 {
7419 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7420 AssertRC(rc);
7421 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7422 if ( !fBlockInt
7423 && !fBlockSti
7424 && !fBlockMovSS)
7425 {
7426 uint8_t u8Interrupt;
7427 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7428 if (RT_SUCCESS(rc))
7429 {
7430 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7431 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7432 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7433
7434 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7435 }
7436 else
7437 {
7438 /** @todo Does this actually happen? If not turn it into an assertion. */
7439 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7440 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7441 }
7442 }
7443 else
7444 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7445 }
7446}
7447
7448
7449/**
7450 * Sets a pending-debug exception to be delivered to the guest if the guest is
7451 * single-stepping.
7452 *
7453 * @param pVCpu Pointer to the VMCPU.
7454 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7455 * out-of-sync. Make sure to update the required fields
7456 * before using them.
7457 */
7458DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7459{
7460 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7461 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7462 {
7463 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7464 AssertRC(rc);
7465 }
7466}
7467
7468
7469/**
7470 * Injects any pending events into the guest if the guest is in a state to
7471 * receive them.
7472 *
7473 * @returns VBox status code (informational status codes included).
7474 * @param pVCpu Pointer to the VMCPU.
7475 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7476 * out-of-sync. Make sure to update the required fields
7477 * before using them.
7478 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7479 * return VINF_EM_DBG_STEPPED if the event was
7480 * dispatched directly.
7481 */
7482static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7483{
7484 HMVMX_ASSERT_PREEMPT_SAFE();
7485 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7486
7487 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7488 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7489 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7490 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7491
7492 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7493 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7494 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7495 Assert(!TRPMHasTrap(pVCpu));
7496
7497 int rc = VINF_SUCCESS;
7498 if (pVCpu->hm.s.Event.fPending)
7499 {
7500 /*
7501 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7502 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7503 * ended up enabling interrupts outside VT-x.
7504 */
7505 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7506 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7507 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7508 {
7509 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7510 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7511 }
7512
7513#ifdef VBOX_STRICT
7514 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7515 {
7516 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7517 Assert(!fBlockInt);
7518 Assert(!fBlockSti);
7519 Assert(!fBlockMovSS);
7520 }
7521 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7522 {
7523 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7524 Assert(!fBlockSti);
7525 Assert(!fBlockMovSS);
7526 Assert(!fBlockNmi);
7527 }
7528#endif
7529 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7530 (uint8_t)uIntType));
7531 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7532 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7533 AssertRCReturn(rc, rc);
7534
7535 /* Update the interruptibility-state as it could have been changed by
7536 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7537 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7538 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7539
7540#ifdef VBOX_WITH_STATISTICS
7541 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7542 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7543 else
7544 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7545#endif
7546 }
7547
7548 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7549 if ( fBlockSti
7550 || fBlockMovSS)
7551 {
7552 if ( !pVCpu->hm.s.fSingleInstruction
7553 && !DBGFIsStepping(pVCpu))
7554 {
7555 /*
7556 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7557 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7558 * See Intel spec. 27.3.4 "Saving Non-Register State".
7559 */
7560 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7561 AssertRCReturn(rc2, rc2);
7562 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7563 }
7564 else if (pMixedCtx->eflags.Bits.u1TF)
7565 {
7566 /*
7567 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7568 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7569 */
7570 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7571 uIntrState = 0;
7572 }
7573 }
7574
7575 /*
7576 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7577 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7578 */
7579 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7580 AssertRC(rc2);
7581
7582 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7583 NOREF(fBlockMovSS); NOREF(fBlockSti);
7584 return rc;
7585}
7586
7587
7588/**
7589 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7590 *
7591 * @param pVCpu Pointer to the VMCPU.
7592 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7593 * out-of-sync. Make sure to update the required fields
7594 * before using them.
7595 */
7596DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7597{
7598 NOREF(pMixedCtx);
7599 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7600 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7601}
7602
7603
7604/**
7605 * Injects a double-fault (#DF) exception into the VM.
7606 *
7607 * @returns VBox status code (informational status code included).
7608 * @param pVCpu Pointer to the VMCPU.
7609 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7610 * out-of-sync. Make sure to update the required fields
7611 * before using them.
7612 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7613 * and should return VINF_EM_DBG_STEPPED if the event
7614 * is injected directly (register modified by us, not
7615 * by hardware on VM-entry).
7616 * @param puIntrState Pointer to the current guest interruptibility-state.
7617 * This interruptibility-state will be updated if
7618 * necessary. This cannot not be NULL.
7619 */
7620DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7621{
7622 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7623 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7624 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7625 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7626 fStepping, puIntrState);
7627}
7628
7629
7630/**
7631 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7632 *
7633 * @param pVCpu Pointer to the VMCPU.
7634 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7635 * out-of-sync. Make sure to update the required fields
7636 * before using them.
7637 */
7638DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7639{
7640 NOREF(pMixedCtx);
7641 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7642 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7643 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7644}
7645
7646
7647/**
7648 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7649 *
7650 * @param pVCpu Pointer to the VMCPU.
7651 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7652 * out-of-sync. Make sure to update the required fields
7653 * before using them.
7654 * @param cbInstr The value of RIP that is to be pushed on the guest
7655 * stack.
7656 */
7657DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7658{
7659 NOREF(pMixedCtx);
7660 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7661 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7662 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7663}
7664
7665
7666/**
7667 * Injects a general-protection (#GP) fault into the VM.
7668 *
7669 * @returns VBox status code (informational status code included).
7670 * @param pVCpu Pointer to the VMCPU.
7671 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7672 * out-of-sync. Make sure to update the required fields
7673 * before using them.
7674 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7675 * mode, i.e. in real-mode it's not valid).
7676 * @param u32ErrorCode The error code associated with the #GP.
7677 * @param fStepping Whether we're running in
7678 * hmR0VmxRunGuestCodeStep() and should return
7679 * VINF_EM_DBG_STEPPED if the event is injected
7680 * directly (register modified by us, not by
7681 * hardware on VM-entry).
7682 * @param puIntrState Pointer to the current guest interruptibility-state.
7683 * This interruptibility-state will be updated if
7684 * necessary. This cannot not be NULL.
7685 */
7686DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7687 bool fStepping, uint32_t *puIntrState)
7688{
7689 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7690 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7691 if (fErrorCodeValid)
7692 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7693 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7694 fStepping, puIntrState);
7695}
7696
7697
7698/**
7699 * Sets a general-protection (#GP) exception as pending-for-injection into the
7700 * VM.
7701 *
7702 * @param pVCpu Pointer to the VMCPU.
7703 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7704 * out-of-sync. Make sure to update the required fields
7705 * before using them.
7706 * @param u32ErrorCode The error code associated with the #GP.
7707 */
7708DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7709{
7710 NOREF(pMixedCtx);
7711 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7712 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7713 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7714 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7715}
7716
7717
7718/**
7719 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7720 *
7721 * @param pVCpu Pointer to the VMCPU.
7722 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7723 * out-of-sync. Make sure to update the required fields
7724 * before using them.
7725 * @param uVector The software interrupt vector number.
7726 * @param cbInstr The value of RIP that is to be pushed on the guest
7727 * stack.
7728 */
7729DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7730{
7731 NOREF(pMixedCtx);
7732 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7733 if ( uVector == X86_XCPT_BP
7734 || uVector == X86_XCPT_OF)
7735 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7736 else
7737 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7738 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7739}
7740
7741
7742/**
7743 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7744 * stack.
7745 *
7746 * @returns VBox status code (information status code included).
7747 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7748 * @param pVM Pointer to the VM.
7749 * @param pMixedCtx Pointer to the guest-CPU context.
7750 * @param uValue The value to push to the guest stack.
7751 */
7752DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7753{
7754 /*
7755 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7756 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7757 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7758 */
7759 if (pMixedCtx->sp == 1)
7760 return VINF_EM_RESET;
7761 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7762 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7763 AssertRCReturn(rc, rc);
7764 return rc;
7765}
7766
7767
7768/**
7769 * Injects an event into the guest upon VM-entry by updating the relevant fields
7770 * in the VM-entry area in the VMCS.
7771 *
7772 * @returns VBox status code (informational error codes included).
7773 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7774 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7775 *
7776 * @param pVCpu Pointer to the VMCPU.
7777 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7778 * be out-of-sync. Make sure to update the required
7779 * fields before using them.
7780 * @param u64IntInfo The VM-entry interruption-information field.
7781 * @param cbInstr The VM-entry instruction length in bytes (for
7782 * software interrupts, exceptions and privileged
7783 * software exceptions).
7784 * @param u32ErrCode The VM-entry exception error code.
7785 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7786 * @param puIntrState Pointer to the current guest interruptibility-state.
7787 * This interruptibility-state will be updated if
7788 * necessary. This cannot not be NULL.
7789 * @param fStepping Whether we're running in
7790 * hmR0VmxRunGuestCodeStep() and should return
7791 * VINF_EM_DBG_STEPPED if the event is injected
7792 * directly (register modified by us, not by
7793 * hardware on VM-entry).
7794 *
7795 * @remarks Requires CR0!
7796 * @remarks No-long-jump zone!!!
7797 */
7798static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7799 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7800{
7801 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7802 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7803 Assert(puIntrState);
7804 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7805
7806 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7807 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7808
7809#ifdef VBOX_STRICT
7810 /* Validate the error-code-valid bit for hardware exceptions. */
7811 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7812 {
7813 switch (uVector)
7814 {
7815 case X86_XCPT_PF:
7816 case X86_XCPT_DF:
7817 case X86_XCPT_TS:
7818 case X86_XCPT_NP:
7819 case X86_XCPT_SS:
7820 case X86_XCPT_GP:
7821 case X86_XCPT_AC:
7822 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7823 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7824 /* fallthru */
7825 default:
7826 break;
7827 }
7828 }
7829#endif
7830
7831 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7832 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7833 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7834
7835 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7836
7837 /* We require CR0 to check if the guest is in real-mode. */
7838 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7839 AssertRCReturn(rc, rc);
7840
7841 /*
7842 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7843 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7844 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7845 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7846 */
7847 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7848 {
7849 PVM pVM = pVCpu->CTX_SUFF(pVM);
7850 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7851 {
7852 Assert(PDMVmmDevHeapIsEnabled(pVM));
7853 Assert(pVM->hm.s.vmx.pRealModeTSS);
7854
7855 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7856 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7857 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7858 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7859 AssertRCReturn(rc, rc);
7860 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7861
7862 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7863 size_t const cbIdtEntry = sizeof(X86IDTR16);
7864 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7865 {
7866 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7867 if (uVector == X86_XCPT_DF)
7868 return VINF_EM_RESET;
7869
7870 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7871 if (uVector == X86_XCPT_GP)
7872 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7873
7874 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7875 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7876 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7877 fStepping, puIntrState);
7878 }
7879
7880 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7881 uint16_t uGuestIp = pMixedCtx->ip;
7882 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7883 {
7884 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7885 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7886 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7887 }
7888 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7889 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7890
7891 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7892 X86IDTR16 IdtEntry;
7893 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7894 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7895 AssertRCReturn(rc, rc);
7896
7897 /* Construct the stack frame for the interrupt/exception handler. */
7898 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7899 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7900 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7901 AssertRCReturn(rc, rc);
7902
7903 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7904 if (rc == VINF_SUCCESS)
7905 {
7906 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7907 pMixedCtx->rip = IdtEntry.offSel;
7908 pMixedCtx->cs.Sel = IdtEntry.uSel;
7909 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7910 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7911 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7912 && uVector == X86_XCPT_PF)
7913 pMixedCtx->cr2 = GCPtrFaultAddress;
7914
7915 /* If any other guest-state bits are changed here, make sure to update
7916 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7917 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7918 | HM_CHANGED_GUEST_RIP
7919 | HM_CHANGED_GUEST_RFLAGS
7920 | HM_CHANGED_GUEST_RSP);
7921
7922 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7923 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7924 {
7925 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7926 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7927 Log4(("Clearing inhibition due to STI.\n"));
7928 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7929 }
7930 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7931 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7932
7933 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7934 it, if we are returning to ring-3 before executing guest code. */
7935 pVCpu->hm.s.Event.fPending = false;
7936
7937 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7938 if (fStepping)
7939 rc = VINF_EM_DBG_STEPPED;
7940 }
7941 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7942 return rc;
7943 }
7944
7945 /*
7946 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7947 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7948 */
7949 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7950 }
7951
7952 /* Validate. */
7953 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7954 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7955 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7956
7957 /* Inject. */
7958 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7959 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7960 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7961 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7962
7963 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7964 && uVector == X86_XCPT_PF)
7965 pMixedCtx->cr2 = GCPtrFaultAddress;
7966
7967 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7968 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7969
7970 AssertRCReturn(rc, rc);
7971 return rc;
7972}
7973
7974
7975/**
7976 * Clears the interrupt-window exiting control in the VMCS and if necessary
7977 * clears the current event in the VMCS as well.
7978 *
7979 * @returns VBox status code.
7980 * @param pVCpu Pointer to the VMCPU.
7981 *
7982 * @remarks Use this function only to clear events that have not yet been
7983 * delivered to the guest but are injected in the VMCS!
7984 * @remarks No-long-jump zone!!!
7985 */
7986static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7987{
7988 int rc;
7989 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7990
7991 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7992 {
7993 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7994 Assert(!pVCpu->hm.s.Event.fPending);
7995 }
7996
7997 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7998 {
7999 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8000 Assert(!pVCpu->hm.s.Event.fPending);
8001 }
8002
8003 if (!pVCpu->hm.s.Event.fPending)
8004 return;
8005
8006#ifdef VBOX_STRICT
8007 uint32_t u32EntryInfo;
8008 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8009 AssertRC(rc);
8010 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8011#endif
8012
8013 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8014 AssertRC(rc);
8015
8016 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8017 AssertRC(rc);
8018
8019 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8020 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8021}
8022
8023
8024/**
8025 * Enters the VT-x session.
8026 *
8027 * @returns VBox status code.
8028 * @param pVM Pointer to the VM.
8029 * @param pVCpu Pointer to the VMCPU.
8030 * @param pCpu Pointer to the CPU info struct.
8031 */
8032VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8033{
8034 AssertPtr(pVM);
8035 AssertPtr(pVCpu);
8036 Assert(pVM->hm.s.vmx.fSupported);
8037 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8038 NOREF(pCpu); NOREF(pVM);
8039
8040 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8041 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8042
8043#ifdef VBOX_STRICT
8044 /* Make sure we're in VMX root mode. */
8045 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8046 if (!(u32HostCR4 & X86_CR4_VMXE))
8047 {
8048 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8049 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8050 }
8051#endif
8052
8053 /*
8054 * Load the VCPU's VMCS as the current (and active) one.
8055 */
8056 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8057 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8058 if (RT_FAILURE(rc))
8059 return rc;
8060
8061 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8062 pVCpu->hm.s.fLeaveDone = false;
8063 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8064
8065 return VINF_SUCCESS;
8066}
8067
8068
8069/**
8070 * The thread-context callback (only on platforms which support it).
8071 *
8072 * @param enmEvent The thread-context event.
8073 * @param pVCpu Pointer to the VMCPU.
8074 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8075 * @thread EMT(pVCpu)
8076 */
8077VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8078{
8079 NOREF(fGlobalInit);
8080
8081 switch (enmEvent)
8082 {
8083 case RTTHREADCTXEVENT_PREEMPTING:
8084 {
8085 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8086 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8087 VMCPU_ASSERT_EMT(pVCpu);
8088
8089 PVM pVM = pVCpu->CTX_SUFF(pVM);
8090 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8091
8092 /* No longjmps (logger flushes, locks) in this fragile context. */
8093 VMMRZCallRing3Disable(pVCpu);
8094 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8095
8096 /*
8097 * Restore host-state (FPU, debug etc.)
8098 */
8099 if (!pVCpu->hm.s.fLeaveDone)
8100 {
8101 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8102 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8103 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8104 pVCpu->hm.s.fLeaveDone = true;
8105 }
8106
8107 /* Leave HM context, takes care of local init (term). */
8108 int rc = HMR0LeaveCpu(pVCpu);
8109 AssertRC(rc); NOREF(rc);
8110
8111 /* Restore longjmp state. */
8112 VMMRZCallRing3Enable(pVCpu);
8113 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8114 break;
8115 }
8116
8117 case RTTHREADCTXEVENT_RESUMED:
8118 {
8119 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8120 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8121 VMCPU_ASSERT_EMT(pVCpu);
8122
8123 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8124 VMMRZCallRing3Disable(pVCpu);
8125 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8126
8127 /* Initialize the bare minimum state required for HM. This takes care of
8128 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8129 int rc = HMR0EnterCpu(pVCpu);
8130 AssertRC(rc);
8131 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8132
8133 /* Load the active VMCS as the current one. */
8134 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8135 {
8136 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8137 AssertRC(rc); NOREF(rc);
8138 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8139 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8140 }
8141 pVCpu->hm.s.fLeaveDone = false;
8142
8143 /* Restore longjmp state. */
8144 VMMRZCallRing3Enable(pVCpu);
8145 break;
8146 }
8147
8148 default:
8149 break;
8150 }
8151}
8152
8153
8154/**
8155 * Saves the host state in the VMCS host-state.
8156 * Sets up the VM-exit MSR-load area.
8157 *
8158 * The CPU state will be loaded from these fields on every successful VM-exit.
8159 *
8160 * @returns VBox status code.
8161 * @param pVM Pointer to the VM.
8162 * @param pVCpu Pointer to the VMCPU.
8163 *
8164 * @remarks No-long-jump zone!!!
8165 */
8166static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8167{
8168 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8169
8170 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8171 return VINF_SUCCESS;
8172
8173 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8174 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8175
8176 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8177 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8178
8179 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8180 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8181
8182 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8183 return rc;
8184}
8185
8186
8187/**
8188 * Saves the host state in the VMCS host-state.
8189 *
8190 * @returns VBox status code.
8191 * @param pVM Pointer to the VM.
8192 * @param pVCpu Pointer to the VMCPU.
8193 *
8194 * @remarks No-long-jump zone!!!
8195 */
8196VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8197{
8198 AssertPtr(pVM);
8199 AssertPtr(pVCpu);
8200
8201 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8202
8203 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8204 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8205 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8206 return hmR0VmxSaveHostState(pVM, pVCpu);
8207}
8208
8209
8210/**
8211 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8212 * loaded from these fields on every successful VM-entry.
8213 *
8214 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8215 * Sets up the VM-entry controls.
8216 * Sets up the appropriate VMX non-root function to execute guest code based on
8217 * the guest CPU mode.
8218 *
8219 * @returns VBox status code.
8220 * @param pVM Pointer to the VM.
8221 * @param pVCpu Pointer to the VMCPU.
8222 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8223 * out-of-sync. Make sure to update the required fields
8224 * before using them.
8225 *
8226 * @remarks No-long-jump zone!!!
8227 */
8228static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8229{
8230 AssertPtr(pVM);
8231 AssertPtr(pVCpu);
8232 AssertPtr(pMixedCtx);
8233 HMVMX_ASSERT_PREEMPT_SAFE();
8234
8235 VMMRZCallRing3Disable(pVCpu);
8236 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8237
8238 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8239
8240 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8241
8242 /* Determine real-on-v86 mode. */
8243 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8244 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8245 && CPUMIsGuestInRealModeEx(pMixedCtx))
8246 {
8247 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8248 }
8249
8250 /*
8251 * Load the guest-state into the VMCS.
8252 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8253 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8254 */
8255 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8256 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8257
8258 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8259 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8260 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8261
8262 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8263 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8264 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8265
8266 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8267 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8268
8269 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8270 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8271
8272 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8273 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8274 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8275
8276 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8277 determine we don't have to swap EFER after all. */
8278 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8279 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8280
8281 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8282 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8283
8284 /*
8285 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8286 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8287 */
8288 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8289 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8290
8291 /* Clear any unused and reserved bits. */
8292 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8293
8294 VMMRZCallRing3Enable(pVCpu);
8295
8296 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8297 return rc;
8298}
8299
8300
8301/**
8302 * Loads the state shared between the host and guest into the VMCS.
8303 *
8304 * @param pVM Pointer to the VM.
8305 * @param pVCpu Pointer to the VMCPU.
8306 * @param pCtx Pointer to the guest-CPU context.
8307 *
8308 * @remarks No-long-jump zone!!!
8309 */
8310static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8311{
8312 NOREF(pVM);
8313
8314 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8315 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8316
8317 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8318 {
8319 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8320 AssertRC(rc);
8321 }
8322
8323 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8324 {
8325 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8326 AssertRC(rc);
8327
8328 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8329 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8330 {
8331 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8332 AssertRC(rc);
8333 }
8334 }
8335
8336 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8337 {
8338#if HC_ARCH_BITS == 64
8339 if (pVM->hm.s.fAllow64BitGuests)
8340 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8341#endif
8342 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8343 }
8344
8345 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8346 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8347}
8348
8349
8350/**
8351 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8352 *
8353 * @param pVM Pointer to the VM.
8354 * @param pVCpu Pointer to the VMCPU.
8355 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8356 * out-of-sync. Make sure to update the required fields
8357 * before using them.
8358 */
8359DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8360{
8361 HMVMX_ASSERT_PREEMPT_SAFE();
8362
8363 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8364#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8365 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8366#endif
8367
8368 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8369 {
8370 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8371 AssertRC(rc);
8372 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8373 }
8374 else if (HMCPU_CF_VALUE(pVCpu))
8375 {
8376 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8377 AssertRC(rc);
8378 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8379 }
8380
8381 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8382 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8383 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8384 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8385}
8386
8387
8388/**
8389 * Does the preparations before executing guest code in VT-x.
8390 *
8391 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8392 * recompiler/IEM. We must be cautious what we do here regarding committing
8393 * guest-state information into the VMCS assuming we assuredly execute the
8394 * guest in VT-x mode.
8395 *
8396 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8397 * the common-state (TRPM/forceflags), we must undo those changes so that the
8398 * recompiler/IEM can (and should) use them when it resumes guest execution.
8399 * Otherwise such operations must be done when we can no longer exit to ring-3.
8400 *
8401 * @returns Strict VBox status code.
8402 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8403 * have been disabled.
8404 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8405 * double-fault into the guest.
8406 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8407 * dispatched directly.
8408 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8409 *
8410 * @param pVM Pointer to the VM.
8411 * @param pVCpu Pointer to the VMCPU.
8412 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8413 * out-of-sync. Make sure to update the required fields
8414 * before using them.
8415 * @param pVmxTransient Pointer to the VMX transient structure.
8416 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8417 * us ignore some of the reasons for returning to
8418 * ring-3, and return VINF_EM_DBG_STEPPED if event
8419 * dispatching took place.
8420 */
8421static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8422{
8423 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8424
8425#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8426 PGMRZDynMapFlushAutoSet(pVCpu);
8427#endif
8428
8429 /* Check force flag actions that might require us to go back to ring-3. */
8430 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8431 if (rc != VINF_SUCCESS)
8432 return rc;
8433
8434#ifndef IEM_VERIFICATION_MODE_FULL
8435 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8436 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8437 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8438 {
8439 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8440 RTGCPHYS GCPhysApicBase;
8441 GCPhysApicBase = pMixedCtx->msrApicBase;
8442 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8443
8444 /* Unalias any existing mapping. */
8445 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8446 AssertRCReturn(rc, rc);
8447
8448 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8449 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8450 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8451 AssertRCReturn(rc, rc);
8452
8453 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8454 }
8455#endif /* !IEM_VERIFICATION_MODE_FULL */
8456
8457 if (TRPMHasTrap(pVCpu))
8458 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8459 else if (!pVCpu->hm.s.Event.fPending)
8460 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8461
8462 /*
8463 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8464 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8465 */
8466 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8467 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8468 {
8469 Assert(rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
8470 return rc;
8471 }
8472
8473 /*
8474 * Load the guest state bits, we can handle longjmps/getting preempted here.
8475 *
8476 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8477 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8478 * Hence, this needs to be done -after- injection of events.
8479 */
8480 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8481
8482 /*
8483 * No longjmps to ring-3 from this point on!!!
8484 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8485 * This also disables flushing of the R0-logger instance (if any).
8486 */
8487 VMMRZCallRing3Disable(pVCpu);
8488
8489 /*
8490 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8491 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8492 *
8493 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8494 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8495 *
8496 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8497 * executing guest code.
8498 */
8499 pVmxTransient->uEflags = ASMIntDisableFlags();
8500 if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8501 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8502 && ( !fStepping /* Optimized for the non-stepping case, of course. */
8503 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8504 {
8505 hmR0VmxClearEventVmcs(pVCpu);
8506 ASMSetFlags(pVmxTransient->uEflags);
8507 VMMRZCallRing3Enable(pVCpu);
8508 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8509 return VINF_EM_RAW_TO_R3;
8510 }
8511
8512 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8513 {
8514 hmR0VmxClearEventVmcs(pVCpu);
8515 ASMSetFlags(pVmxTransient->uEflags);
8516 VMMRZCallRing3Enable(pVCpu);
8517 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8518 return VINF_EM_RAW_INTERRUPT;
8519 }
8520
8521 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8522 pVCpu->hm.s.Event.fPending = false;
8523
8524 return VINF_SUCCESS;
8525}
8526
8527
8528/**
8529 * Prepares to run guest code in VT-x and we've committed to doing so. This
8530 * means there is no backing out to ring-3 or anywhere else at this
8531 * point.
8532 *
8533 * @param pVM Pointer to the VM.
8534 * @param pVCpu Pointer to the VMCPU.
8535 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8536 * out-of-sync. Make sure to update the required fields
8537 * before using them.
8538 * @param pVmxTransient Pointer to the VMX transient structure.
8539 *
8540 * @remarks Called with preemption disabled.
8541 * @remarks No-long-jump zone!!!
8542 */
8543static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8544{
8545 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8546 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8547 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8548
8549 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8550 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8551
8552#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8553 if (!CPUMIsGuestFPUStateActive(pVCpu))
8554 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8555 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8556#endif
8557
8558 if ( pVCpu->hm.s.fUseGuestFpu
8559 && !CPUMIsGuestFPUStateActive(pVCpu))
8560 {
8561 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8562 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8563 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8564 }
8565
8566 /*
8567 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8568 */
8569 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8570 && pVCpu->hm.s.vmx.cMsrs > 0)
8571 {
8572 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8573 }
8574
8575 /*
8576 * Load the host state bits as we may've been preempted (only happens when
8577 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8578 */
8579 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8580 {
8581 /* This ASSUMES that pfnStartVM has been set up already. */
8582 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8583 AssertRC(rc);
8584 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8585 }
8586 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8587
8588 /*
8589 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8590 */
8591 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8592 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8593 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8594
8595 /* Store status of the shared guest-host state at the time of VM-entry. */
8596#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8597 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8598 {
8599 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8600 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8601 }
8602 else
8603#endif
8604 {
8605 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8606 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8607 }
8608 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8609
8610 /*
8611 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8612 */
8613 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8614 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8615
8616 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8617 RTCPUID idCurrentCpu = pCpu->idCpu;
8618 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8619 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8620 {
8621 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8622 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8623 }
8624
8625 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8626 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8627 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8628 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8629
8630 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8631
8632 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8633 to start executing. */
8634
8635 /*
8636 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8637 */
8638 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8639 {
8640 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8641 {
8642 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8643 AssertRC(rc2);
8644 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8645 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8646 true /* fUpdateHostMsr */);
8647 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8648 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8649 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8650 }
8651 else
8652 {
8653 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8654 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8655 }
8656 }
8657
8658#ifdef VBOX_STRICT
8659 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8660 hmR0VmxCheckHostEferMsr(pVCpu);
8661 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8662#endif
8663#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8664 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8665 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8666 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8667#endif
8668}
8669
8670
8671/**
8672 * Performs some essential restoration of state after running guest code in
8673 * VT-x.
8674 *
8675 * @param pVM Pointer to the VM.
8676 * @param pVCpu Pointer to the VMCPU.
8677 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8678 * out-of-sync. Make sure to update the required fields
8679 * before using them.
8680 * @param pVmxTransient Pointer to the VMX transient structure.
8681 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8682 *
8683 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8684 *
8685 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8686 * unconditionally when it is safe to do so.
8687 */
8688static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8689{
8690 NOREF(pVM);
8691
8692 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8693
8694 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8695 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8696 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8697 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8698 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8699 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8700
8701 /** @todo Last-seen-tick shouldn't be necessary when TM supports invariant
8702 * mode. */
8703 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8704 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8705
8706 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8707 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8708 Assert(!(ASMGetFlags() & X86_EFL_IF));
8709 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8710
8711#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8712 if (CPUMIsGuestFPUStateActive(pVCpu))
8713 {
8714 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8715 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8716 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8717 }
8718#endif
8719
8720#if HC_ARCH_BITS == 64
8721 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8722#endif
8723 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8724#ifdef VBOX_STRICT
8725 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8726#endif
8727 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8728 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8729
8730 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8731 uint32_t uExitReason;
8732 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8733 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8734 AssertRC(rc);
8735 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8736 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8737
8738 /* Update the VM-exit history array. */
8739 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8740
8741 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8742 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8743 {
8744 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8745 pVmxTransient->fVMEntryFailed));
8746 return;
8747 }
8748
8749 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8750 {
8751 /** @todo We can optimize this by only syncing with our force-flags when
8752 * really needed and keeping the VMCS state as it is for most
8753 * VM-exits. */
8754 /* Update the guest interruptibility-state from the VMCS. */
8755 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8756
8757#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8758 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8759 AssertRC(rc);
8760#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8761 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8762 AssertRC(rc);
8763#endif
8764
8765 /*
8766 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8767 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8768 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8769 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8770 */
8771 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8772 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8773 {
8774 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8775 AssertRC(rc);
8776 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8777 }
8778 }
8779}
8780
8781
8782/**
8783 * Runs the guest code using VT-x the normal way.
8784 *
8785 * @returns VBox status code.
8786 * @param pVM Pointer to the VM.
8787 * @param pVCpu Pointer to the VMCPU.
8788 * @param pCtx Pointer to the guest-CPU context.
8789 *
8790 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8791 */
8792static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8793{
8794 VMXTRANSIENT VmxTransient;
8795 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8796 int rc = VERR_INTERNAL_ERROR_5;
8797 uint32_t cLoops = 0;
8798
8799 for (;; cLoops++)
8800 {
8801 Assert(!HMR0SuspendPending());
8802 HMVMX_ASSERT_CPU_SAFE();
8803
8804 /* Preparatory work for running guest code, this may force us to return
8805 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8806 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8807 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8808 if (rc != VINF_SUCCESS)
8809 break;
8810
8811 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8812 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8813 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8814
8815 /* Restore any residual host-state and save any bits shared between host
8816 and guest into the guest-CPU state. Re-enables interrupts! */
8817 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8818
8819 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8820 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8821 {
8822 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8823 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8824 return rc;
8825 }
8826
8827 /* Handle the VM-exit. */
8828 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8829 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8830 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8831 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8832 HMVMX_START_EXIT_DISPATCH_PROF();
8833#ifdef HMVMX_USE_FUNCTION_TABLE
8834 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8835#else
8836 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8837#endif
8838 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8839 if (rc != VINF_SUCCESS)
8840 break;
8841 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8842 {
8843 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8844 rc = VINF_EM_RAW_INTERRUPT;
8845 break;
8846 }
8847 }
8848
8849 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8850 return rc;
8851}
8852
8853
8854/**
8855 * Single steps guest code using VT-x.
8856 *
8857 * @returns VBox status code.
8858 * @param pVM Pointer to the VM.
8859 * @param pVCpu Pointer to the VMCPU.
8860 * @param pCtx Pointer to the guest-CPU context.
8861 *
8862 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8863 */
8864static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8865{
8866 VMXTRANSIENT VmxTransient;
8867 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8868 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8869 uint32_t cLoops = 0;
8870 uint16_t uCsStart = pCtx->cs.Sel;
8871 uint64_t uRipStart = pCtx->rip;
8872
8873 for (;; cLoops++)
8874 {
8875 Assert(!HMR0SuspendPending());
8876 HMVMX_ASSERT_CPU_SAFE();
8877
8878 /* Preparatory work for running guest code, this may force us to return
8879 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8880 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8881 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8882 if (rcStrict != VINF_SUCCESS)
8883 break;
8884
8885 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8886 rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8887 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8888
8889 /* Restore any residual host-state and save any bits shared between host
8890 and guest into the guest-CPU state. Re-enables interrupts! */
8891 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8892
8893 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8894 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
8895 {
8896 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8897 hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
8898 return VBOXSTRICTRC_TODO(rcStrict);
8899 }
8900
8901 /* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8902 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8903 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8904 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8905 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8906 HMVMX_START_EXIT_DISPATCH_PROF();
8907 rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8908 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8909 if (rcStrict != VINF_SUCCESS)
8910 break;
8911 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8912 {
8913 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8914 rcStrict = VINF_EM_RAW_INTERRUPT;
8915 break;
8916 }
8917
8918 /*
8919 * Did the RIP change, if so, consider it a single step.
8920 * Otherwise, make sure one of the TFs gets set.
8921 */
8922 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8923 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8924 AssertRCReturn(rc2, rc2);
8925 if ( pCtx->rip != uRipStart
8926 || pCtx->cs.Sel != uCsStart)
8927 {
8928 rcStrict = VINF_EM_DBG_STEPPED;
8929 break;
8930 }
8931 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8932 }
8933
8934 /*
8935 * Clear the X86_EFL_TF if necessary.
8936 */
8937 if (pVCpu->hm.s.fClearTrapFlag)
8938 {
8939 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8940 AssertRCReturn(rc2, rc2);
8941 pVCpu->hm.s.fClearTrapFlag = false;
8942 pCtx->eflags.Bits.u1TF = 0;
8943 }
8944 /** @todo there seems to be issues with the resume flag when the monitor trap
8945 * flag is pending without being used. Seen early in bios init when
8946 * accessing APIC page in protected mode. */
8947
8948 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8949 return VBOXSTRICTRC_TODO(rcStrict);
8950}
8951
8952
8953/**
8954 * Runs the guest code using VT-x.
8955 *
8956 * @returns VBox status code.
8957 * @param pVM Pointer to the VM.
8958 * @param pVCpu Pointer to the VMCPU.
8959 * @param pCtx Pointer to the guest-CPU context.
8960 */
8961VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8962{
8963 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8964 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8965 HMVMX_ASSERT_PREEMPT_SAFE();
8966
8967 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8968
8969 int rc;
8970 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8971 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8972 else
8973 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8974
8975 if (rc == VERR_EM_INTERPRETER)
8976 rc = VINF_EM_RAW_EMULATE_INSTR;
8977 else if (rc == VINF_EM_RESET)
8978 rc = VINF_EM_TRIPLE_FAULT;
8979
8980 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8981 if (RT_FAILURE(rc2))
8982 {
8983 pVCpu->hm.s.u32HMError = rc;
8984 rc = rc2;
8985 }
8986 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8987 return rc;
8988}
8989
8990
8991#ifndef HMVMX_USE_FUNCTION_TABLE
8992DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8993{
8994#ifdef DEBUG_ramshankar
8995# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8996# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8997#endif
8998 int rc;
8999 switch (rcReason)
9000 {
9001 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9002 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9003 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9004 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9005 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9006 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9007 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9008 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9009 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9010 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9011 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9012 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9013 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9014 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9015 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9016 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9017 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9018 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9019 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9020 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9021 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9022 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9023 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9024 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9025 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9026 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9027 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9028 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9029 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9030 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9031 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9032 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9033 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9034 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9035
9036 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9037 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9038 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9039 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9040 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9041 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9042 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9043 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9044 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9045
9046 case VMX_EXIT_VMCLEAR:
9047 case VMX_EXIT_VMLAUNCH:
9048 case VMX_EXIT_VMPTRLD:
9049 case VMX_EXIT_VMPTRST:
9050 case VMX_EXIT_VMREAD:
9051 case VMX_EXIT_VMRESUME:
9052 case VMX_EXIT_VMWRITE:
9053 case VMX_EXIT_VMXOFF:
9054 case VMX_EXIT_VMXON:
9055 case VMX_EXIT_INVEPT:
9056 case VMX_EXIT_INVVPID:
9057 case VMX_EXIT_VMFUNC:
9058 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9059 break;
9060 default:
9061 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9062 break;
9063 }
9064 return rc;
9065}
9066#endif /* !HMVMX_USE_FUNCTION_TABLE */
9067
9068
9069/**
9070 * Single-stepping VM-exit filtering.
9071 *
9072 * This is preprocessing the exits and deciding whether we've gotten far enough
9073 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9074 * performed.
9075 *
9076 * @returns Strict VBox status code.
9077 * @param pVCpu The virtual CPU of the calling EMT.
9078 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9079 * out-of-sync. Make sure to update the required
9080 * fields before using them.
9081 * @param pVmxTransient Pointer to the VMX-transient structure.
9082 * @param uExitReason The VM-exit reason.
9083 */
9084DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9085 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9086{
9087 switch (uExitReason)
9088 {
9089 case VMX_EXIT_XCPT_OR_NMI:
9090 {
9091 /* Check for host NMI. */
9092 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9093 AssertRCReturn(rc2, rc2);
9094 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9095 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9096 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9097 /* fall thru */
9098 }
9099
9100 case VMX_EXIT_EPT_MISCONFIG:
9101 case VMX_EXIT_TRIPLE_FAULT:
9102 case VMX_EXIT_APIC_ACCESS:
9103 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9104 case VMX_EXIT_TASK_SWITCH:
9105
9106 /* Instruction specific VM-exits: */
9107 case VMX_EXIT_IO_INSTR:
9108 case VMX_EXIT_CPUID:
9109 case VMX_EXIT_RDTSC:
9110 case VMX_EXIT_RDTSCP:
9111 case VMX_EXIT_MOV_CRX:
9112 case VMX_EXIT_MWAIT:
9113 case VMX_EXIT_MONITOR:
9114 case VMX_EXIT_RDMSR:
9115 case VMX_EXIT_WRMSR:
9116 case VMX_EXIT_MOV_DRX:
9117 case VMX_EXIT_HLT:
9118 case VMX_EXIT_INVD:
9119 case VMX_EXIT_INVLPG:
9120 case VMX_EXIT_RSM:
9121 case VMX_EXIT_PAUSE:
9122 case VMX_EXIT_XDTR_ACCESS:
9123 case VMX_EXIT_TR_ACCESS:
9124 case VMX_EXIT_WBINVD:
9125 case VMX_EXIT_XSETBV:
9126 case VMX_EXIT_RDRAND:
9127 case VMX_EXIT_INVPCID:
9128 case VMX_EXIT_GETSEC:
9129 case VMX_EXIT_RDPMC:
9130 case VMX_EXIT_VMCALL:
9131 case VMX_EXIT_VMCLEAR:
9132 case VMX_EXIT_VMLAUNCH:
9133 case VMX_EXIT_VMPTRLD:
9134 case VMX_EXIT_VMPTRST:
9135 case VMX_EXIT_VMREAD:
9136 case VMX_EXIT_VMRESUME:
9137 case VMX_EXIT_VMWRITE:
9138 case VMX_EXIT_VMXOFF:
9139 case VMX_EXIT_VMXON:
9140 case VMX_EXIT_INVEPT:
9141 case VMX_EXIT_INVVPID:
9142 case VMX_EXIT_VMFUNC:
9143 {
9144 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9145 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9146 AssertRCReturn(rc2, rc2);
9147 if ( pMixedCtx->rip != uRipStart
9148 || pMixedCtx->cs.Sel != uCsStart)
9149 return VINF_EM_DBG_STEPPED;
9150 break;
9151 }
9152 }
9153
9154 /*
9155 * Normal processing.
9156 */
9157#ifdef HMVMX_USE_FUNCTION_TABLE
9158 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9159#else
9160 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9161#endif
9162}
9163
9164
9165#ifdef DEBUG
9166/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9167# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9168 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9169
9170# define HMVMX_ASSERT_PREEMPT_CPUID() \
9171 do \
9172 { \
9173 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9174 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9175 } while (0)
9176
9177# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9178 do { \
9179 AssertPtr(pVCpu); \
9180 AssertPtr(pMixedCtx); \
9181 AssertPtr(pVmxTransient); \
9182 Assert(pVmxTransient->fVMEntryFailed == false); \
9183 Assert(ASMIntAreEnabled()); \
9184 HMVMX_ASSERT_PREEMPT_SAFE(); \
9185 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9186 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9187 HMVMX_ASSERT_PREEMPT_SAFE(); \
9188 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9189 HMVMX_ASSERT_PREEMPT_CPUID(); \
9190 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9191 } while (0)
9192
9193# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9194 do { \
9195 Log4Func(("\n")); \
9196 } while (0)
9197#else /* Release builds */
9198# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9199 do { \
9200 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9201 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9202 } while (0)
9203# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9204#endif
9205
9206
9207/**
9208 * Advances the guest RIP after reading it from the VMCS.
9209 *
9210 * @returns VBox status code.
9211 * @param pVCpu Pointer to the VMCPU.
9212 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9213 * out-of-sync. Make sure to update the required fields
9214 * before using them.
9215 * @param pVmxTransient Pointer to the VMX transient structure.
9216 *
9217 * @remarks No-long-jump zone!!!
9218 */
9219DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9220{
9221 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9222 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9223 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9224 AssertRCReturn(rc, rc);
9225
9226 pMixedCtx->rip += pVmxTransient->cbInstr;
9227 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9228
9229 /*
9230 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9231 * pending debug exception field as it takes care of priority of events.
9232 *
9233 * See Intel spec. 32.2.1 "Debug Exceptions".
9234 */
9235 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9236
9237 return rc;
9238}
9239
9240
9241/**
9242 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9243 * and update error record fields accordingly.
9244 *
9245 * @return VMX_IGS_* return codes.
9246 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9247 * wrong with the guest state.
9248 *
9249 * @param pVM Pointer to the VM.
9250 * @param pVCpu Pointer to the VMCPU.
9251 * @param pCtx Pointer to the guest-CPU state.
9252 *
9253 * @remarks This function assumes our cache of the VMCS controls
9254 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9255 */
9256static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9257{
9258#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9259#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9260 uError = (err); \
9261 break; \
9262 } else do { } while (0)
9263
9264 int rc;
9265 uint32_t uError = VMX_IGS_ERROR;
9266 uint32_t u32Val;
9267 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9268
9269 do
9270 {
9271 /*
9272 * CR0.
9273 */
9274 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9275 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9276 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9277 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9278 if (fUnrestrictedGuest)
9279 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9280
9281 uint32_t u32GuestCR0;
9282 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9283 AssertRCBreak(rc);
9284 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9285 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9286 if ( !fUnrestrictedGuest
9287 && (u32GuestCR0 & X86_CR0_PG)
9288 && !(u32GuestCR0 & X86_CR0_PE))
9289 {
9290 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9291 }
9292
9293 /*
9294 * CR4.
9295 */
9296 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9297 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9298
9299 uint32_t u32GuestCR4;
9300 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9301 AssertRCBreak(rc);
9302 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9303 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9304
9305 /*
9306 * IA32_DEBUGCTL MSR.
9307 */
9308 uint64_t u64Val;
9309 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9310 AssertRCBreak(rc);
9311 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9312 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9313 {
9314 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9315 }
9316 uint64_t u64DebugCtlMsr = u64Val;
9317
9318#ifdef VBOX_STRICT
9319 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9320 AssertRCBreak(rc);
9321 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9322#endif
9323 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9324
9325 /*
9326 * RIP and RFLAGS.
9327 */
9328 uint32_t u32Eflags;
9329#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9330 if (HMVMX_IS_64BIT_HOST_MODE())
9331 {
9332 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9333 AssertRCBreak(rc);
9334 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9335 if ( !fLongModeGuest
9336 || !pCtx->cs.Attr.n.u1Long)
9337 {
9338 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9339 }
9340 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9341 * must be identical if the "IA-32e mode guest" VM-entry
9342 * control is 1 and CS.L is 1. No check applies if the
9343 * CPU supports 64 linear-address bits. */
9344
9345 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9346 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9347 AssertRCBreak(rc);
9348 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9349 VMX_IGS_RFLAGS_RESERVED);
9350 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9351 u32Eflags = u64Val;
9352 }
9353 else
9354#endif
9355 {
9356 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9357 AssertRCBreak(rc);
9358 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9359 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9360 }
9361
9362 if ( fLongModeGuest
9363 || ( fUnrestrictedGuest
9364 && !(u32GuestCR0 & X86_CR0_PE)))
9365 {
9366 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9367 }
9368
9369 uint32_t u32EntryInfo;
9370 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9371 AssertRCBreak(rc);
9372 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9373 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9374 {
9375 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9376 }
9377
9378 /*
9379 * 64-bit checks.
9380 */
9381#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9382 if (HMVMX_IS_64BIT_HOST_MODE())
9383 {
9384 if ( fLongModeGuest
9385 && !fUnrestrictedGuest)
9386 {
9387 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9388 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9389 }
9390
9391 if ( !fLongModeGuest
9392 && (u32GuestCR4 & X86_CR4_PCIDE))
9393 {
9394 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9395 }
9396
9397 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9398 * 51:32 beyond the processor's physical-address width are 0. */
9399
9400 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9401 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9402 {
9403 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9404 }
9405
9406 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9407 AssertRCBreak(rc);
9408 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9409
9410 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9411 AssertRCBreak(rc);
9412 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9413 }
9414#endif
9415
9416 /*
9417 * PERF_GLOBAL MSR.
9418 */
9419 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9420 {
9421 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9422 AssertRCBreak(rc);
9423 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9424 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9425 }
9426
9427 /*
9428 * PAT MSR.
9429 */
9430 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9431 {
9432 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9433 AssertRCBreak(rc);
9434 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9435 for (unsigned i = 0; i < 8; i++)
9436 {
9437 uint8_t u8Val = (u64Val & 0xff);
9438 if ( u8Val != 0 /* UC */
9439 && u8Val != 1 /* WC */
9440 && u8Val != 4 /* WT */
9441 && u8Val != 5 /* WP */
9442 && u8Val != 6 /* WB */
9443 && u8Val != 7 /* UC- */)
9444 {
9445 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9446 }
9447 u64Val >>= 8;
9448 }
9449 }
9450
9451 /*
9452 * EFER MSR.
9453 */
9454 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9455 {
9456 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9457 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9458 AssertRCBreak(rc);
9459 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9460 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9461 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9462 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9463 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9464 || !(u32GuestCR0 & X86_CR0_PG)
9465 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9466 VMX_IGS_EFER_LMA_LME_MISMATCH);
9467 }
9468
9469 /*
9470 * Segment registers.
9471 */
9472 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9473 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9474 if (!(u32Eflags & X86_EFL_VM))
9475 {
9476 /* CS */
9477 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9478 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9479 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9480 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9481 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9482 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9483 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9484 /* CS cannot be loaded with NULL in protected mode. */
9485 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9486 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9487 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9488 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9489 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9490 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9491 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9492 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9493 else
9494 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9495
9496 /* SS */
9497 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9498 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9499 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9500 if ( !(pCtx->cr0 & X86_CR0_PE)
9501 || pCtx->cs.Attr.n.u4Type == 3)
9502 {
9503 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9504 }
9505 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9506 {
9507 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9508 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9509 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9510 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9511 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9512 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9513 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9514 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9515 }
9516
9517 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9518 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9519 {
9520 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9521 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9522 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9523 || pCtx->ds.Attr.n.u4Type > 11
9524 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9525 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9526 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9527 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9528 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9529 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9530 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9531 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9532 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9533 }
9534 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9535 {
9536 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9537 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9538 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9539 || pCtx->es.Attr.n.u4Type > 11
9540 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9541 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9542 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9543 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9544 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9545 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9546 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9547 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9548 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9549 }
9550 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9551 {
9552 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9553 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9554 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9555 || pCtx->fs.Attr.n.u4Type > 11
9556 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9557 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9558 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9559 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9560 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9561 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9562 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9563 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9564 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9565 }
9566 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9567 {
9568 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9569 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9570 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9571 || pCtx->gs.Attr.n.u4Type > 11
9572 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9573 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9574 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9575 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9576 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9577 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9578 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9579 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9580 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9581 }
9582 /* 64-bit capable CPUs. */
9583#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9584 if (HMVMX_IS_64BIT_HOST_MODE())
9585 {
9586 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9587 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9588 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9589 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9590 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9591 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9592 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9593 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9594 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9595 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9596 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9597 }
9598#endif
9599 }
9600 else
9601 {
9602 /* V86 mode checks. */
9603 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9604 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9605 {
9606 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9607 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9608 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9609 }
9610 else
9611 {
9612 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9613 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9614 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9615 }
9616
9617 /* CS */
9618 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9619 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9620 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9621 /* SS */
9622 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9623 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9624 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9625 /* DS */
9626 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9627 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9628 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9629 /* ES */
9630 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9631 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9632 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9633 /* FS */
9634 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9635 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9636 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9637 /* GS */
9638 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9639 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9640 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9641 /* 64-bit capable CPUs. */
9642#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9643 if (HMVMX_IS_64BIT_HOST_MODE())
9644 {
9645 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9646 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9647 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9648 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9649 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9650 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9651 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9652 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9653 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9654 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9655 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9656 }
9657#endif
9658 }
9659
9660 /*
9661 * TR.
9662 */
9663 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9664 /* 64-bit capable CPUs. */
9665#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9666 if (HMVMX_IS_64BIT_HOST_MODE())
9667 {
9668 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9669 }
9670#endif
9671 if (fLongModeGuest)
9672 {
9673 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9674 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9675 }
9676 else
9677 {
9678 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9679 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9680 VMX_IGS_TR_ATTR_TYPE_INVALID);
9681 }
9682 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9683 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9684 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9685 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9686 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9687 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9688 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9689 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9690
9691 /*
9692 * GDTR and IDTR.
9693 */
9694#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9695 if (HMVMX_IS_64BIT_HOST_MODE())
9696 {
9697 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9698 AssertRCBreak(rc);
9699 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9700
9701 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9702 AssertRCBreak(rc);
9703 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9704 }
9705#endif
9706
9707 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9708 AssertRCBreak(rc);
9709 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9710
9711 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9712 AssertRCBreak(rc);
9713 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9714
9715 /*
9716 * Guest Non-Register State.
9717 */
9718 /* Activity State. */
9719 uint32_t u32ActivityState;
9720 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9721 AssertRCBreak(rc);
9722 HMVMX_CHECK_BREAK( !u32ActivityState
9723 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9724 VMX_IGS_ACTIVITY_STATE_INVALID);
9725 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9726 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9727 uint32_t u32IntrState;
9728 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9729 AssertRCBreak(rc);
9730 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9731 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9732 {
9733 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9734 }
9735
9736 /** @todo Activity state and injecting interrupts. Left as a todo since we
9737 * currently don't use activity states but ACTIVE. */
9738
9739 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9740 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9741
9742 /* Guest interruptibility-state. */
9743 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9744 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9745 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9746 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9747 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9748 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9749 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9750 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9751 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9752 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9753 {
9754 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9755 {
9756 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9757 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9758 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9759 }
9760 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9761 {
9762 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9763 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9764 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9765 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9766 }
9767 }
9768 /** @todo Assumes the processor is not in SMM. */
9769 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9770 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9771 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9772 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9773 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9774 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9775 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9776 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9777 {
9778 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9779 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9780 }
9781
9782 /* Pending debug exceptions. */
9783 if (HMVMX_IS_64BIT_HOST_MODE())
9784 {
9785 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9786 AssertRCBreak(rc);
9787 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9788 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9789 u32Val = u64Val; /* For pending debug exceptions checks below. */
9790 }
9791 else
9792 {
9793 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9794 AssertRCBreak(rc);
9795 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9796 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9797 }
9798
9799 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9800 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9801 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9802 {
9803 if ( (u32Eflags & X86_EFL_TF)
9804 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9805 {
9806 /* Bit 14 is PendingDebug.BS. */
9807 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9808 }
9809 if ( !(u32Eflags & X86_EFL_TF)
9810 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9811 {
9812 /* Bit 14 is PendingDebug.BS. */
9813 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9814 }
9815 }
9816
9817 /* VMCS link pointer. */
9818 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9819 AssertRCBreak(rc);
9820 if (u64Val != UINT64_C(0xffffffffffffffff))
9821 {
9822 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9823 /** @todo Bits beyond the processor's physical-address width MBZ. */
9824 /** @todo 32-bit located in memory referenced by value of this field (as a
9825 * physical address) must contain the processor's VMCS revision ID. */
9826 /** @todo SMM checks. */
9827 }
9828
9829 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9830 * not using Nested Paging? */
9831 if ( pVM->hm.s.fNestedPaging
9832 && !fLongModeGuest
9833 && CPUMIsGuestInPAEModeEx(pCtx))
9834 {
9835 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9836 AssertRCBreak(rc);
9837 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9838
9839 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9840 AssertRCBreak(rc);
9841 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9842
9843 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9844 AssertRCBreak(rc);
9845 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9846
9847 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9848 AssertRCBreak(rc);
9849 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9850 }
9851
9852 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9853 if (uError == VMX_IGS_ERROR)
9854 uError = VMX_IGS_REASON_NOT_FOUND;
9855 } while (0);
9856
9857 pVCpu->hm.s.u32HMError = uError;
9858 return uError;
9859
9860#undef HMVMX_ERROR_BREAK
9861#undef HMVMX_CHECK_BREAK
9862}
9863
9864/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9865/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9866/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9867
9868/** @name VM-exit handlers.
9869 * @{
9870 */
9871
9872/**
9873 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9874 */
9875HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9876{
9877 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9878 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9879 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9880 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9881 return VINF_SUCCESS;
9882 return VINF_EM_RAW_INTERRUPT;
9883}
9884
9885
9886/**
9887 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9888 */
9889HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9890{
9891 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9892 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9893
9894 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9895 AssertRCReturn(rc, rc);
9896
9897 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9898 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9899 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9900 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9901
9902 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9903 {
9904 /*
9905 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9906 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9907 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9908 *
9909 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9910 */
9911 VMXDispatchHostNmi();
9912 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9913 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9914 return VINF_SUCCESS;
9915 }
9916
9917 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9918 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9919 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9920 {
9921 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9922 return VINF_SUCCESS;
9923 }
9924 if (RT_UNLIKELY(rc == VINF_EM_RESET))
9925 {
9926 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9927 return rc;
9928 }
9929
9930 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9931 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9932 switch (uIntType)
9933 {
9934 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9935 Assert(uVector == X86_XCPT_DB);
9936 /* no break */
9937 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9938 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9939 /* no break */
9940 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9941 {
9942 switch (uVector)
9943 {
9944 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9945 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9946 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9947 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9948 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9949 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9950#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9951 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9952 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9953 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9954 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9955 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9956 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9957 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9958 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9959 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9960 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9961 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9962 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9963#endif
9964 default:
9965 {
9966 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9967 AssertRCReturn(rc, rc);
9968
9969 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9970 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9971 {
9972 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9973 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9974 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9975
9976 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9977 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9978 AssertRCReturn(rc, rc);
9979 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9980 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9981 0 /* GCPtrFaultAddress */);
9982 AssertRCReturn(rc, rc);
9983 }
9984 else
9985 {
9986 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9987 pVCpu->hm.s.u32HMError = uVector;
9988 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9989 }
9990 break;
9991 }
9992 }
9993 break;
9994 }
9995
9996 default:
9997 {
9998 pVCpu->hm.s.u32HMError = uExitIntInfo;
9999 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10000 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10001 break;
10002 }
10003 }
10004 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10005 return rc;
10006}
10007
10008
10009/**
10010 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10011 */
10012HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10013{
10014 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10015
10016 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
10017 hmR0VmxClearIntWindowExitVmcs(pVCpu);
10018
10019 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10020 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
10021 return VINF_SUCCESS;
10022}
10023
10024
10025/**
10026 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
10027 */
10028HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10029{
10030 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10031 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
10032 {
10033 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
10034 HMVMX_RETURN_UNEXPECTED_EXIT();
10035 }
10036
10037 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
10038
10039 /*
10040 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
10041 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
10042 */
10043 uint32_t uIntrState = 0;
10044 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10045 AssertRCReturn(rc, rc);
10046
10047 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
10048 if ( fBlockSti
10049 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
10050 {
10051 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10052 }
10053
10054 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
10055 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
10056
10057 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10058 return VINF_SUCCESS;
10059}
10060
10061
10062/**
10063 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10064 */
10065HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10066{
10067 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10068 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10069 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10070}
10071
10072
10073/**
10074 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10075 */
10076HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10077{
10078 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10079 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10080 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10081}
10082
10083
10084/**
10085 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10086 */
10087HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10088{
10089 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10090 PVM pVM = pVCpu->CTX_SUFF(pVM);
10091 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10092 if (RT_LIKELY(rc == VINF_SUCCESS))
10093 {
10094 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10095 Assert(pVmxTransient->cbInstr == 2);
10096 }
10097 else
10098 {
10099 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10100 rc = VERR_EM_INTERPRETER;
10101 }
10102 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10103 return rc;
10104}
10105
10106
10107/**
10108 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10109 */
10110HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10111{
10112 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10113 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10114 AssertRCReturn(rc, rc);
10115
10116 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10117 return VINF_EM_RAW_EMULATE_INSTR;
10118
10119 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10120 HMVMX_RETURN_UNEXPECTED_EXIT();
10121}
10122
10123
10124/**
10125 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10126 */
10127HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10128{
10129 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10130 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10131 AssertRCReturn(rc, rc);
10132
10133 PVM pVM = pVCpu->CTX_SUFF(pVM);
10134 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10135 if (RT_LIKELY(rc == VINF_SUCCESS))
10136 {
10137 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10138 Assert(pVmxTransient->cbInstr == 2);
10139 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10140 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10141 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10142 }
10143 else
10144 rc = VERR_EM_INTERPRETER;
10145 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10146 return rc;
10147}
10148
10149
10150/**
10151 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10152 */
10153HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10154{
10155 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10156 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10157 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10158 AssertRCReturn(rc, rc);
10159
10160 PVM pVM = pVCpu->CTX_SUFF(pVM);
10161 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10162 if (RT_LIKELY(rc == VINF_SUCCESS))
10163 {
10164 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10165 Assert(pVmxTransient->cbInstr == 3);
10166 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10167 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10168 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10169 }
10170 else
10171 {
10172 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10173 rc = VERR_EM_INTERPRETER;
10174 }
10175 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10176 return rc;
10177}
10178
10179
10180/**
10181 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10182 */
10183HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10184{
10185 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10186 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10187 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10188 AssertRCReturn(rc, rc);
10189
10190 PVM pVM = pVCpu->CTX_SUFF(pVM);
10191 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10192 if (RT_LIKELY(rc == VINF_SUCCESS))
10193 {
10194 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10195 Assert(pVmxTransient->cbInstr == 2);
10196 }
10197 else
10198 {
10199 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10200 rc = VERR_EM_INTERPRETER;
10201 }
10202 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10203 return rc;
10204}
10205
10206
10207/**
10208 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10209 */
10210HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10211{
10212 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10213
10214 int rc = VERR_NOT_SUPPORTED;
10215 if (GIMAreHypercallsEnabled(pVCpu))
10216 {
10217 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10218 AssertRCReturn(rc, rc);
10219
10220 rc = GIMHypercall(pVCpu, pMixedCtx);
10221 }
10222 if (rc != VINF_SUCCESS)
10223 {
10224 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10225 rc = VINF_SUCCESS;
10226 }
10227
10228 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10229 return rc;
10230}
10231
10232
10233/**
10234 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10235 */
10236HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10237{
10238 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10239 PVM pVM = pVCpu->CTX_SUFF(pVM);
10240 Assert(!pVM->hm.s.fNestedPaging);
10241
10242 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10243 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10244 AssertRCReturn(rc, rc);
10245
10246 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10247 rc = VBOXSTRICTRC_VAL(rc2);
10248 if (RT_LIKELY(rc == VINF_SUCCESS))
10249 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10250 else
10251 {
10252 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10253 pVmxTransient->uExitQualification, rc));
10254 }
10255 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10256 return rc;
10257}
10258
10259
10260/**
10261 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10262 */
10263HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10264{
10265 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10266 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10267 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10268 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10269 AssertRCReturn(rc, rc);
10270
10271 PVM pVM = pVCpu->CTX_SUFF(pVM);
10272 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10273 if (RT_LIKELY(rc == VINF_SUCCESS))
10274 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10275 else
10276 {
10277 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10278 rc = VERR_EM_INTERPRETER;
10279 }
10280 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10281 return rc;
10282}
10283
10284
10285/**
10286 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10287 */
10288HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10289{
10290 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10291 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10292 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10293 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10294 AssertRCReturn(rc, rc);
10295
10296 PVM pVM = pVCpu->CTX_SUFF(pVM);
10297 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10298 rc = VBOXSTRICTRC_VAL(rc2);
10299 if (RT_LIKELY( rc == VINF_SUCCESS
10300 || rc == VINF_EM_HALT))
10301 {
10302 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10303 AssertRCReturn(rc3, rc3);
10304
10305 if ( rc == VINF_EM_HALT
10306 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10307 {
10308 rc = VINF_SUCCESS;
10309 }
10310 }
10311 else
10312 {
10313 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10314 rc = VERR_EM_INTERPRETER;
10315 }
10316 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10317 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10318 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10319 return rc;
10320}
10321
10322
10323/**
10324 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10325 */
10326HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10327{
10328 /*
10329 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10330 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10331 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10332 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10333 */
10334 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10335 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10336 HMVMX_RETURN_UNEXPECTED_EXIT();
10337}
10338
10339
10340/**
10341 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10342 */
10343HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10344{
10345 /*
10346 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10347 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10348 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10349 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10350 */
10351 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10352 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10353 HMVMX_RETURN_UNEXPECTED_EXIT();
10354}
10355
10356
10357/**
10358 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10359 */
10360HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10361{
10362 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10363 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10364 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10365 HMVMX_RETURN_UNEXPECTED_EXIT();
10366}
10367
10368
10369/**
10370 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10371 */
10372HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10373{
10374 /*
10375 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10376 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10377 * See Intel spec. 25.3 "Other Causes of VM-exits".
10378 */
10379 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10380 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10381 HMVMX_RETURN_UNEXPECTED_EXIT();
10382}
10383
10384
10385/**
10386 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10387 * VM-exit.
10388 */
10389HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10390{
10391 /*
10392 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10393 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10394 *
10395 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10396 * See Intel spec. "23.8 Restrictions on VMX operation".
10397 */
10398 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10399 return VINF_SUCCESS;
10400}
10401
10402
10403/**
10404 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10405 * VM-exit.
10406 */
10407HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10408{
10409 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10410 return VINF_EM_RESET;
10411}
10412
10413
10414/**
10415 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10416 */
10417HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10418{
10419 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10420 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10421 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10422 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10423 AssertRCReturn(rc, rc);
10424
10425 pMixedCtx->rip++;
10426 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10427 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10428 rc = VINF_SUCCESS;
10429 else
10430 rc = VINF_EM_HALT;
10431
10432 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10433 return rc;
10434}
10435
10436
10437/**
10438 * VM-exit handler for instructions that result in a #UD exception delivered to
10439 * the guest.
10440 */
10441HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10442{
10443 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10444 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10445 return VINF_SUCCESS;
10446}
10447
10448
10449/**
10450 * VM-exit handler for expiry of the VMX preemption timer.
10451 */
10452HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10453{
10454 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10455
10456 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10457 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10458
10459 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10460 PVM pVM = pVCpu->CTX_SUFF(pVM);
10461 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10462 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10463 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10464}
10465
10466
10467/**
10468 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10469 */
10470HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10471{
10472 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10473
10474 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10475 /** @todo check if XSETBV is supported by the recompiler. */
10476 return VERR_EM_INTERPRETER;
10477}
10478
10479
10480/**
10481 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10482 */
10483HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10484{
10485 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10486
10487 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10488 /** @todo implement EMInterpretInvpcid() */
10489 return VERR_EM_INTERPRETER;
10490}
10491
10492
10493/**
10494 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10495 * Error VM-exit.
10496 */
10497HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10498{
10499 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10500 AssertRCReturn(rc, rc);
10501
10502 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10503 AssertRCReturn(rc, rc);
10504
10505 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10506 NOREF(uInvalidReason);
10507
10508#ifdef VBOX_STRICT
10509 uint32_t uIntrState;
10510 HMVMXHCUINTREG uHCReg;
10511 uint64_t u64Val;
10512 uint32_t u32Val;
10513
10514 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10515 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10516 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10517 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10518 AssertRCReturn(rc, rc);
10519
10520 Log4(("uInvalidReason %u\n", uInvalidReason));
10521 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10522 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10523 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10524 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10525
10526 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10527 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10528 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10529 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10530 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10531 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10532 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10533 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10534 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10535 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10536 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10537 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10538#else
10539 NOREF(pVmxTransient);
10540#endif
10541
10542 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10543 return VERR_VMX_INVALID_GUEST_STATE;
10544}
10545
10546
10547/**
10548 * VM-exit handler for VM-entry failure due to an MSR-load
10549 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10550 */
10551HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10552{
10553 NOREF(pVmxTransient);
10554 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10555 HMVMX_RETURN_UNEXPECTED_EXIT();
10556}
10557
10558
10559/**
10560 * VM-exit handler for VM-entry failure due to a machine-check event
10561 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10562 */
10563HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10564{
10565 NOREF(pVmxTransient);
10566 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10567 HMVMX_RETURN_UNEXPECTED_EXIT();
10568}
10569
10570
10571/**
10572 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10573 * theory.
10574 */
10575HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10576{
10577 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10578 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10579 return VERR_VMX_UNDEFINED_EXIT_CODE;
10580}
10581
10582
10583/**
10584 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10585 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10586 * Conditional VM-exit.
10587 */
10588HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10589{
10590 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10591
10592 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10593 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10594 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10595 return VERR_EM_INTERPRETER;
10596 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10597 HMVMX_RETURN_UNEXPECTED_EXIT();
10598}
10599
10600
10601/**
10602 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10603 */
10604HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10605{
10606 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10607
10608 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10609 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10610 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10611 return VERR_EM_INTERPRETER;
10612 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10613 HMVMX_RETURN_UNEXPECTED_EXIT();
10614}
10615
10616
10617/**
10618 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10619 */
10620HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10621{
10622 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10623
10624 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10625 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10626 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10627 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10628 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10629 {
10630 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10631 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10632 }
10633 AssertRCReturn(rc, rc);
10634 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10635
10636#ifdef VBOX_STRICT
10637 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10638 {
10639 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10640 && pMixedCtx->ecx != MSR_K6_EFER)
10641 {
10642 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10643 HMVMX_RETURN_UNEXPECTED_EXIT();
10644 }
10645# if HC_ARCH_BITS == 64
10646 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10647 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10648 {
10649 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10650 HMVMX_RETURN_UNEXPECTED_EXIT();
10651 }
10652# endif
10653 }
10654#endif
10655
10656 PVM pVM = pVCpu->CTX_SUFF(pVM);
10657 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10658 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10659 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10660 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10661 if (RT_LIKELY(rc == VINF_SUCCESS))
10662 {
10663 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10664 Assert(pVmxTransient->cbInstr == 2);
10665 }
10666 return rc;
10667}
10668
10669
10670/**
10671 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10672 */
10673HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10674{
10675 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10676 PVM pVM = pVCpu->CTX_SUFF(pVM);
10677 int rc = VINF_SUCCESS;
10678
10679 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10680 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10681 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10682 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10683 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10684 {
10685 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10686 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10687 }
10688 AssertRCReturn(rc, rc);
10689 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10690
10691 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10692 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10693 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10694
10695 if (RT_LIKELY(rc == VINF_SUCCESS))
10696 {
10697 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10698
10699 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10700 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10701 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10702 {
10703 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10704 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10705 EMInterpretWrmsr() changes it. */
10706 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10707 }
10708 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10709 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10710 else if (pMixedCtx->ecx == MSR_K6_EFER)
10711 {
10712 /*
10713 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10714 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10715 * the other bits as well, SCE and NXE. See @bugref{7368}.
10716 */
10717 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10718 }
10719
10720 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10721 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10722 {
10723 switch (pMixedCtx->ecx)
10724 {
10725 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10726 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10727 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10728 case MSR_K8_FS_BASE: /* no break */
10729 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10730 case MSR_K6_EFER: /* already handled above */ break;
10731 default:
10732 {
10733 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10734 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10735#if HC_ARCH_BITS == 64
10736 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10737 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10738#endif
10739 break;
10740 }
10741 }
10742 }
10743#ifdef VBOX_STRICT
10744 else
10745 {
10746 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10747 switch (pMixedCtx->ecx)
10748 {
10749 case MSR_IA32_SYSENTER_CS:
10750 case MSR_IA32_SYSENTER_EIP:
10751 case MSR_IA32_SYSENTER_ESP:
10752 case MSR_K8_FS_BASE:
10753 case MSR_K8_GS_BASE:
10754 {
10755 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10756 HMVMX_RETURN_UNEXPECTED_EXIT();
10757 }
10758
10759 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10760 default:
10761 {
10762 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10763 {
10764 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10765 if (pMixedCtx->ecx != MSR_K6_EFER)
10766 {
10767 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10768 pMixedCtx->ecx));
10769 HMVMX_RETURN_UNEXPECTED_EXIT();
10770 }
10771 }
10772
10773#if HC_ARCH_BITS == 64
10774 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10775 {
10776 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10777 HMVMX_RETURN_UNEXPECTED_EXIT();
10778 }
10779#endif
10780 break;
10781 }
10782 }
10783 }
10784#endif /* VBOX_STRICT */
10785 }
10786 return rc;
10787}
10788
10789
10790/**
10791 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10792 */
10793HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10794{
10795 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10796
10797 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10798 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10799 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10800 return VERR_EM_INTERPRETER;
10801 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10802 HMVMX_RETURN_UNEXPECTED_EXIT();
10803}
10804
10805
10806/**
10807 * VM-exit handler for when the TPR value is lowered below the specified
10808 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10809 */
10810HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10811{
10812 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10813 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10814
10815 /*
10816 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10817 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10818 * resume guest execution.
10819 */
10820 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10821 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10822 return VINF_SUCCESS;
10823}
10824
10825
10826/**
10827 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10828 * VM-exit.
10829 *
10830 * @retval VINF_SUCCESS when guest execution can continue.
10831 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10832 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10833 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10834 * recompiler.
10835 */
10836HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10837{
10838 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10839 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10840 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10841 AssertRCReturn(rc, rc);
10842
10843 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10844 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10845 PVM pVM = pVCpu->CTX_SUFF(pVM);
10846 switch (uAccessType)
10847 {
10848 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10849 {
10850#if 0
10851 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10852 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10853#else
10854 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10855 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10856 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10857#endif
10858 AssertRCReturn(rc, rc);
10859
10860 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10861 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10862 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10863 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10864
10865 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10866 {
10867 case 0: /* CR0 */
10868 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10869 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10870 break;
10871 case 2: /* CR2 */
10872 /* Nothing to do here, CR2 it's not part of the VMCS. */
10873 break;
10874 case 3: /* CR3 */
10875 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10876 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10877 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10878 break;
10879 case 4: /* CR4 */
10880 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10881 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10882 break;
10883 case 8: /* CR8 */
10884 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10885 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10886 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10887 break;
10888 default:
10889 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10890 break;
10891 }
10892
10893 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10894 break;
10895 }
10896
10897 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10898 {
10899 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10900 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10901 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10902 AssertRCReturn(rc, rc);
10903 Assert( !pVM->hm.s.fNestedPaging
10904 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10905 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10906
10907 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10908 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10909 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10910
10911 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10912 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10913 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10914 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10915 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10916 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10917 break;
10918 }
10919
10920 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10921 {
10922 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10923 AssertRCReturn(rc, rc);
10924 rc = EMInterpretCLTS(pVM, pVCpu);
10925 AssertRCReturn(rc, rc);
10926 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10927 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10928 Log4(("CRX CLTS write rc=%d\n", rc));
10929 break;
10930 }
10931
10932 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10933 {
10934 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10935 AssertRCReturn(rc, rc);
10936 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10937 if (RT_LIKELY(rc == VINF_SUCCESS))
10938 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10939 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10940 Log4(("CRX LMSW write rc=%d\n", rc));
10941 break;
10942 }
10943
10944 default:
10945 {
10946 AssertMsgFailed(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType));
10947 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10948 }
10949 }
10950
10951 /* Validate possible error codes. */
10952 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10953 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10954 if (RT_SUCCESS(rc))
10955 {
10956 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10957 AssertRCReturn(rc2, rc2);
10958 }
10959
10960 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10961 return rc;
10962}
10963
10964
10965/**
10966 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10967 * VM-exit.
10968 */
10969HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10970{
10971 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10972 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10973
10974 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10975 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10976 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10977 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10978 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10979 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10980 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10981 AssertRCReturn(rc2, rc2);
10982
10983 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10984 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10985 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10986 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10987 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10988 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10989 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10990 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10991
10992 /* I/O operation lookup arrays. */
10993 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10994 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10995
10996 VBOXSTRICTRC rcStrict;
10997 uint32_t const cbValue = s_aIOSizes[uIOWidth];
10998 uint32_t const cbInstr = pVmxTransient->cbInstr;
10999 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
11000 PVM pVM = pVCpu->CTX_SUFF(pVM);
11001 if (fIOString)
11002 {
11003#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
11004 /*
11005 * INS/OUTS - I/O String instruction.
11006 *
11007 * Use instruction-information if available, otherwise fall back on
11008 * interpreting the instruction.
11009 */
11010 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11011 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
11012 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
11013 {
11014 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11015 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11016 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11017 AssertRCReturn(rc2, rc2);
11018 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
11019 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
11020 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
11021 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
11022 if (fIOWrite)
11023 {
11024 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
11025 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
11026 }
11027 else
11028 {
11029 /*
11030 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
11031 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
11032 * See Intel Instruction spec. for "INS".
11033 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
11034 */
11035 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
11036 }
11037 }
11038 else
11039 {
11040 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11041 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11042 AssertRCReturn(rc2, rc2);
11043 rcStrict = IEMExecOne(pVCpu);
11044 }
11045 /** @todo IEM needs to be setting these flags somehow. */
11046 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11047 fUpdateRipAlready = true;
11048#else
11049 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11050 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
11051 if (RT_SUCCESS(rcStrict))
11052 {
11053 if (fIOWrite)
11054 {
11055 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11056 (DISCPUMODE)pDis->uAddrMode, cbValue);
11057 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
11058 }
11059 else
11060 {
11061 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11062 (DISCPUMODE)pDis->uAddrMode, cbValue);
11063 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11064 }
11065 }
11066 else
11067 {
11068 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
11069 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11070 }
11071#endif
11072 }
11073 else
11074 {
11075 /*
11076 * IN/OUT - I/O instruction.
11077 */
11078 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11079 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11080 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11081 if (fIOWrite)
11082 {
11083 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11084 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11085 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11086 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11087 }
11088 else
11089 {
11090 uint32_t u32Result = 0;
11091 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11092 if (IOM_SUCCESS(rcStrict))
11093 {
11094 /* Save result of I/O IN instr. in AL/AX/EAX. */
11095 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11096 }
11097 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11098 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11099 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11100 }
11101 }
11102
11103 if (IOM_SUCCESS(rcStrict))
11104 {
11105 if (!fUpdateRipAlready)
11106 {
11107 pMixedCtx->rip += cbInstr;
11108 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11109 }
11110
11111 /*
11112 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11113 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11114 */
11115 if (fIOString)
11116 {
11117 /** @todo Single-step for INS/OUTS with REP prefix? */
11118 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11119 }
11120 else if (fStepping)
11121 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11122
11123 /*
11124 * If any I/O breakpoints are armed, we need to check if one triggered
11125 * and take appropriate action.
11126 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11127 */
11128 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11129 AssertRCReturn(rc2, rc2);
11130
11131 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11132 * execution engines about whether hyper BPs and such are pending. */
11133 uint32_t const uDr7 = pMixedCtx->dr[7];
11134 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11135 && X86_DR7_ANY_RW_IO(uDr7)
11136 && (pMixedCtx->cr4 & X86_CR4_DE))
11137 || DBGFBpIsHwIoArmed(pVM)))
11138 {
11139 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11140
11141 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11142 VMMRZCallRing3Disable(pVCpu);
11143 HM_DISABLE_PREEMPT_IF_NEEDED();
11144
11145 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11146
11147 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11148 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11149 {
11150 /* Raise #DB. */
11151 if (fIsGuestDbgActive)
11152 ASMSetDR6(pMixedCtx->dr[6]);
11153 if (pMixedCtx->dr[7] != uDr7)
11154 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11155
11156 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11157 }
11158 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11159 else if ( rcStrict2 != VINF_SUCCESS
11160 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11161 rcStrict = rcStrict2;
11162
11163 HM_RESTORE_PREEMPT_IF_NEEDED();
11164 VMMRZCallRing3Enable(pVCpu);
11165 }
11166 }
11167
11168#ifdef DEBUG
11169 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11170 Assert(!fIOWrite);
11171 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11172 Assert(fIOWrite);
11173 else
11174 {
11175 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11176 * statuses, that the VMM device and some others may return. See
11177 * IOM_SUCCESS() for guidance. */
11178 AssertMsg( RT_FAILURE(rcStrict)
11179 || rcStrict == VINF_SUCCESS
11180 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11181 || rcStrict == VINF_EM_DBG_BREAKPOINT
11182 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11183 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11184 }
11185#endif
11186
11187 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11188 return VBOXSTRICTRC_TODO(rcStrict);
11189}
11190
11191
11192/**
11193 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11194 * VM-exit.
11195 */
11196HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11197{
11198 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11199
11200 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11201 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11202 AssertRCReturn(rc, rc);
11203 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11204 {
11205 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11206 AssertRCReturn(rc, rc);
11207 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11208 {
11209 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11210
11211 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11212 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11213
11214 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11215 Assert(!pVCpu->hm.s.Event.fPending);
11216 pVCpu->hm.s.Event.fPending = true;
11217 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11218 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11219 AssertRCReturn(rc, rc);
11220 if (fErrorCodeValid)
11221 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11222 else
11223 pVCpu->hm.s.Event.u32ErrCode = 0;
11224 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11225 && uVector == X86_XCPT_PF)
11226 {
11227 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11228 }
11229
11230 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11231 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11232 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11233 }
11234 }
11235
11236 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11237 * emulation. */
11238 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11239 return VERR_EM_INTERPRETER;
11240}
11241
11242
11243/**
11244 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11245 */
11246HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11247{
11248 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11249 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11250 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11251 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11252 AssertRCReturn(rc, rc);
11253 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11254 return VINF_EM_DBG_STEPPED;
11255}
11256
11257
11258/**
11259 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11260 */
11261HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11262{
11263 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11264
11265 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11266 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11267 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11268 return VINF_SUCCESS;
11269 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11270 return rc;
11271
11272#if 0
11273 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11274 * just sync the whole thing. */
11275 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11276#else
11277 /* Aggressive state sync. for now. */
11278 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11279 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11280 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11281#endif
11282 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11283 AssertRCReturn(rc, rc);
11284
11285 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11286 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11287 switch (uAccessType)
11288 {
11289 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11290 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11291 {
11292 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11293 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11294 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11295
11296 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11297 GCPhys &= PAGE_BASE_GC_MASK;
11298 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11299 PVM pVM = pVCpu->CTX_SUFF(pVM);
11300 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11301 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11302
11303 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11304 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11305 CPUMCTX2CORE(pMixedCtx), GCPhys);
11306 rc = VBOXSTRICTRC_VAL(rc2);
11307 Log4(("ApicAccess rc=%d\n", rc));
11308 if ( rc == VINF_SUCCESS
11309 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11310 || rc == VERR_PAGE_NOT_PRESENT)
11311 {
11312 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11313 | HM_CHANGED_GUEST_RSP
11314 | HM_CHANGED_GUEST_RFLAGS
11315 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11316 rc = VINF_SUCCESS;
11317 }
11318 break;
11319 }
11320
11321 default:
11322 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11323 rc = VINF_EM_RAW_EMULATE_INSTR;
11324 break;
11325 }
11326
11327 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11328 if (rc != VINF_SUCCESS)
11329 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccessToR3);
11330 return rc;
11331}
11332
11333
11334/**
11335 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11336 * VM-exit.
11337 */
11338HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11339{
11340 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11341
11342 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11343 if (pVmxTransient->fWasGuestDebugStateActive)
11344 {
11345 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11346 HMVMX_RETURN_UNEXPECTED_EXIT();
11347 }
11348
11349 int rc = VERR_INTERNAL_ERROR_5;
11350 if ( !DBGFIsStepping(pVCpu)
11351 && !pVCpu->hm.s.fSingleInstruction
11352 && !pVmxTransient->fWasHyperDebugStateActive)
11353 {
11354 /* Don't intercept MOV DRx and #DB any more. */
11355 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11356 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11357 AssertRCReturn(rc, rc);
11358
11359 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11360 {
11361#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11362 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11363 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11364 AssertRCReturn(rc, rc);
11365#endif
11366 }
11367
11368 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11369 VMMRZCallRing3Disable(pVCpu);
11370 HM_DISABLE_PREEMPT_IF_NEEDED();
11371
11372 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11373 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11374 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11375
11376 HM_RESTORE_PREEMPT_IF_NEEDED();
11377 VMMRZCallRing3Enable(pVCpu);
11378
11379#ifdef VBOX_WITH_STATISTICS
11380 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11381 AssertRCReturn(rc, rc);
11382 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11383 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11384 else
11385 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11386#endif
11387 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11388 return VINF_SUCCESS;
11389 }
11390
11391 /*
11392 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11393 * Update the segment registers and DR7 from the CPU.
11394 */
11395 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11396 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11397 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11398 AssertRCReturn(rc, rc);
11399 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11400
11401 PVM pVM = pVCpu->CTX_SUFF(pVM);
11402 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11403 {
11404 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11405 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11406 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11407 if (RT_SUCCESS(rc))
11408 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11409 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11410 }
11411 else
11412 {
11413 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11414 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11415 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11416 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11417 }
11418
11419 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11420 if (RT_SUCCESS(rc))
11421 {
11422 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11423 AssertRCReturn(rc2, rc2);
11424 }
11425 return rc;
11426}
11427
11428
11429/**
11430 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11431 * Conditional VM-exit.
11432 */
11433HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11434{
11435 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11436 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11437
11438 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11439 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11440 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11441 return VINF_SUCCESS;
11442 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11443 return rc;
11444
11445 RTGCPHYS GCPhys = 0;
11446 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11447
11448#if 0
11449 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11450#else
11451 /* Aggressive state sync. for now. */
11452 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11453 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11454 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11455#endif
11456 AssertRCReturn(rc, rc);
11457
11458 /*
11459 * If we succeed, resume guest execution.
11460 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11461 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11462 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11463 * weird case. See @bugref{6043}.
11464 */
11465 PVM pVM = pVCpu->CTX_SUFF(pVM);
11466 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11467 rc = VBOXSTRICTRC_VAL(rc2);
11468 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11469 if ( rc == VINF_SUCCESS
11470 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11471 || rc == VERR_PAGE_NOT_PRESENT)
11472 {
11473 /* Successfully handled MMIO operation. */
11474 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11475 | HM_CHANGED_GUEST_RSP
11476 | HM_CHANGED_GUEST_RFLAGS
11477 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11478 rc = VINF_SUCCESS;
11479 }
11480 return rc;
11481}
11482
11483
11484/**
11485 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11486 * VM-exit.
11487 */
11488HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11489{
11490 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11491 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11492
11493 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11494 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11495 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11496 return VINF_SUCCESS;
11497 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11498 return rc;
11499
11500 RTGCPHYS GCPhys = 0;
11501 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11502 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11503#if 0
11504 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11505#else
11506 /* Aggressive state sync. for now. */
11507 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11508 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11509 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11510#endif
11511 AssertRCReturn(rc, rc);
11512
11513 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11514 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11515
11516 RTGCUINT uErrorCode = 0;
11517 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11518 uErrorCode |= X86_TRAP_PF_ID;
11519 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11520 uErrorCode |= X86_TRAP_PF_RW;
11521 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11522 uErrorCode |= X86_TRAP_PF_P;
11523
11524 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11525
11526 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11527 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11528
11529 /* Handle the pagefault trap for the nested shadow table. */
11530 PVM pVM = pVCpu->CTX_SUFF(pVM);
11531 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11532 TRPMResetTrap(pVCpu);
11533
11534 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11535 if ( rc == VINF_SUCCESS
11536 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11537 || rc == VERR_PAGE_NOT_PRESENT)
11538 {
11539 /* Successfully synced our nested page tables. */
11540 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11541 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11542 | HM_CHANGED_GUEST_RSP
11543 | HM_CHANGED_GUEST_RFLAGS);
11544 return VINF_SUCCESS;
11545 }
11546
11547 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11548 return rc;
11549}
11550
11551/** @} */
11552
11553/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11554/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11555/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11556
11557/** @name VM-exit exception handlers.
11558 * @{
11559 */
11560
11561/**
11562 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11563 */
11564static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11565{
11566 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11567 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11568
11569 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11570 AssertRCReturn(rc, rc);
11571
11572 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11573 {
11574 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11575 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11576
11577 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11578 * provides VM-exit instruction length. If this causes problem later,
11579 * disassemble the instruction like it's done on AMD-V. */
11580 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11581 AssertRCReturn(rc2, rc2);
11582 return rc;
11583 }
11584
11585 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11586 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11587 return rc;
11588}
11589
11590
11591/**
11592 * VM-exit exception handler for #BP (Breakpoint exception).
11593 */
11594static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11595{
11596 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11597 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11598
11599 /** @todo Try optimize this by not saving the entire guest state unless
11600 * really needed. */
11601 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11602 AssertRCReturn(rc, rc);
11603
11604 PVM pVM = pVCpu->CTX_SUFF(pVM);
11605 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11606 if (rc == VINF_EM_RAW_GUEST_TRAP)
11607 {
11608 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11609 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11610 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11611 AssertRCReturn(rc, rc);
11612
11613 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11614 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11615 }
11616
11617 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11618 return rc;
11619}
11620
11621
11622/**
11623 * VM-exit exception handler for #DB (Debug exception).
11624 */
11625static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11626{
11627 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11628 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11629 Log6(("XcptDB\n"));
11630
11631 /*
11632 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11633 * for processing.
11634 */
11635 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11636 AssertRCReturn(rc, rc);
11637
11638 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11639 uint64_t uDR6 = X86_DR6_INIT_VAL;
11640 uDR6 |= ( pVmxTransient->uExitQualification
11641 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11642
11643 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11644 if (rc == VINF_EM_RAW_GUEST_TRAP)
11645 {
11646 /*
11647 * The exception was for the guest. Update DR6, DR7.GD and
11648 * IA32_DEBUGCTL.LBR before forwarding it.
11649 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11650 */
11651 VMMRZCallRing3Disable(pVCpu);
11652 HM_DISABLE_PREEMPT_IF_NEEDED();
11653
11654 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11655 pMixedCtx->dr[6] |= uDR6;
11656 if (CPUMIsGuestDebugStateActive(pVCpu))
11657 ASMSetDR6(pMixedCtx->dr[6]);
11658
11659 HM_RESTORE_PREEMPT_IF_NEEDED();
11660 VMMRZCallRing3Enable(pVCpu);
11661
11662 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11663 AssertRCReturn(rc, rc);
11664
11665 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11666 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11667
11668 /* Paranoia. */
11669 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11670 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11671
11672 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11673 AssertRCReturn(rc, rc);
11674
11675 /*
11676 * Raise #DB in the guest.
11677 *
11678 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11679 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11680 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11681 *
11682 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11683 */
11684 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11685 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11686 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11687 AssertRCReturn(rc, rc);
11688 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11689 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11690 return VINF_SUCCESS;
11691 }
11692
11693 /*
11694 * Not a guest trap, must be a hypervisor related debug event then.
11695 * Update DR6 in case someone is interested in it.
11696 */
11697 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11698 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11699 CPUMSetHyperDR6(pVCpu, uDR6);
11700
11701 return rc;
11702}
11703
11704
11705/**
11706 * VM-exit exception handler for #NM (Device-not-available exception: floating
11707 * point exception).
11708 */
11709static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11710{
11711 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11712
11713 /* We require CR0 and EFER. EFER is always up-to-date. */
11714 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11715 AssertRCReturn(rc, rc);
11716
11717 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11718 VMMRZCallRing3Disable(pVCpu);
11719 HM_DISABLE_PREEMPT_IF_NEEDED();
11720
11721 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11722 if (pVmxTransient->fWasGuestFPUStateActive)
11723 {
11724 rc = VINF_EM_RAW_GUEST_TRAP;
11725 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11726 }
11727 else
11728 {
11729#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11730 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11731#endif
11732 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11733 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11734 }
11735
11736 HM_RESTORE_PREEMPT_IF_NEEDED();
11737 VMMRZCallRing3Enable(pVCpu);
11738
11739 if (rc == VINF_SUCCESS)
11740 {
11741 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11742 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11743 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11744 pVCpu->hm.s.fUseGuestFpu = true;
11745 }
11746 else
11747 {
11748 /* Forward #NM to the guest. */
11749 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11750 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11751 AssertRCReturn(rc, rc);
11752 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11753 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11754 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11755 }
11756
11757 return VINF_SUCCESS;
11758}
11759
11760
11761/**
11762 * VM-exit exception handler for #GP (General-protection exception).
11763 *
11764 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11765 */
11766static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11767{
11768 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11769 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11770
11771 int rc = VERR_INTERNAL_ERROR_5;
11772 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11773 {
11774#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11775 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11776 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11777 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11778 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11779 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11780 AssertRCReturn(rc, rc);
11781 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11782 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11783 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11784 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11785 return rc;
11786#else
11787 /* We don't intercept #GP. */
11788 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11789 NOREF(pVmxTransient);
11790 return VERR_VMX_UNEXPECTED_EXCEPTION;
11791#endif
11792 }
11793
11794 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11795 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11796
11797 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11798 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11799 AssertRCReturn(rc, rc);
11800
11801 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11802 uint32_t cbOp = 0;
11803 PVM pVM = pVCpu->CTX_SUFF(pVM);
11804 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11805 if (RT_SUCCESS(rc))
11806 {
11807 rc = VINF_SUCCESS;
11808 Assert(cbOp == pDis->cbInstr);
11809 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11810 switch (pDis->pCurInstr->uOpcode)
11811 {
11812 case OP_CLI:
11813 {
11814 pMixedCtx->eflags.Bits.u1IF = 0;
11815 pMixedCtx->eflags.Bits.u1RF = 0;
11816 pMixedCtx->rip += pDis->cbInstr;
11817 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11818 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11819 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11820 break;
11821 }
11822
11823 case OP_STI:
11824 {
11825 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11826 pMixedCtx->eflags.Bits.u1IF = 1;
11827 pMixedCtx->eflags.Bits.u1RF = 0;
11828 pMixedCtx->rip += pDis->cbInstr;
11829 if (!fOldIF)
11830 {
11831 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11832 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11833 }
11834 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11835 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11837 break;
11838 }
11839
11840 case OP_HLT:
11841 {
11842 rc = VINF_EM_HALT;
11843 pMixedCtx->rip += pDis->cbInstr;
11844 pMixedCtx->eflags.Bits.u1RF = 0;
11845 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11846 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11847 break;
11848 }
11849
11850 case OP_POPF:
11851 {
11852 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11853 uint32_t cbParm;
11854 uint32_t uMask;
11855 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11856 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11857 {
11858 cbParm = 4;
11859 uMask = 0xffffffff;
11860 }
11861 else
11862 {
11863 cbParm = 2;
11864 uMask = 0xffff;
11865 }
11866
11867 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11868 RTGCPTR GCPtrStack = 0;
11869 X86EFLAGS Eflags;
11870 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11871 &GCPtrStack);
11872 if (RT_SUCCESS(rc))
11873 {
11874 Assert(sizeof(Eflags.u32) >= cbParm);
11875 Eflags.u32 = 0;
11876 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11877 }
11878 if (RT_FAILURE(rc))
11879 {
11880 rc = VERR_EM_INTERPRETER;
11881 break;
11882 }
11883 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11884 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
11885 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11886 pMixedCtx->esp += cbParm;
11887 pMixedCtx->esp &= uMask;
11888 pMixedCtx->rip += pDis->cbInstr;
11889 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11890 | HM_CHANGED_GUEST_RSP
11891 | HM_CHANGED_GUEST_RFLAGS);
11892 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11893 if (fStepping)
11894 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11895
11896 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11897 break;
11898 }
11899
11900 case OP_PUSHF:
11901 {
11902 uint32_t cbParm;
11903 uint32_t uMask;
11904 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11905 {
11906 cbParm = 4;
11907 uMask = 0xffffffff;
11908 }
11909 else
11910 {
11911 cbParm = 2;
11912 uMask = 0xffff;
11913 }
11914
11915 /* Get the stack pointer & push the contents of eflags onto the stack. */
11916 RTGCPTR GCPtrStack = 0;
11917 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11918 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11919 if (RT_FAILURE(rc))
11920 {
11921 rc = VERR_EM_INTERPRETER;
11922 break;
11923 }
11924 X86EFLAGS Eflags = pMixedCtx->eflags;
11925 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11926 Eflags.Bits.u1RF = 0;
11927 Eflags.Bits.u1VM = 0;
11928
11929 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11930 if (RT_FAILURE(rc))
11931 {
11932 rc = VERR_EM_INTERPRETER;
11933 break;
11934 }
11935 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11936 pMixedCtx->esp -= cbParm;
11937 pMixedCtx->esp &= uMask;
11938 pMixedCtx->rip += pDis->cbInstr;
11939 pMixedCtx->eflags.Bits.u1RF = 0;
11940 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11941 | HM_CHANGED_GUEST_RSP
11942 | HM_CHANGED_GUEST_RFLAGS);
11943 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11944 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11945 break;
11946 }
11947
11948 case OP_IRET:
11949 {
11950 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11951 * instruction reference. */
11952 RTGCPTR GCPtrStack = 0;
11953 uint32_t uMask = 0xffff;
11954 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11955 uint16_t aIretFrame[3];
11956 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11957 {
11958 rc = VERR_EM_INTERPRETER;
11959 break;
11960 }
11961 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11962 &GCPtrStack);
11963 if (RT_SUCCESS(rc))
11964 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11965 if (RT_FAILURE(rc))
11966 {
11967 rc = VERR_EM_INTERPRETER;
11968 break;
11969 }
11970 pMixedCtx->eip = 0;
11971 pMixedCtx->ip = aIretFrame[0];
11972 pMixedCtx->cs.Sel = aIretFrame[1];
11973 pMixedCtx->cs.ValidSel = aIretFrame[1];
11974 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11975 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
11976 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11977 pMixedCtx->sp += sizeof(aIretFrame);
11978 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11979 | HM_CHANGED_GUEST_SEGMENT_REGS
11980 | HM_CHANGED_GUEST_RSP
11981 | HM_CHANGED_GUEST_RFLAGS);
11982 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11983 if (fStepping)
11984 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11985 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11986 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11987 break;
11988 }
11989
11990 case OP_INT:
11991 {
11992 uint16_t uVector = pDis->Param1.uValue & 0xff;
11993 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11994 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11995 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11996 break;
11997 }
11998
11999 case OP_INTO:
12000 {
12001 if (pMixedCtx->eflags.Bits.u1OF)
12002 {
12003 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
12004 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12005 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12006 }
12007 else
12008 {
12009 pMixedCtx->eflags.Bits.u1RF = 0;
12010 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12011 }
12012 break;
12013 }
12014
12015 default:
12016 {
12017 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
12018 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
12019 EMCODETYPE_SUPERVISOR);
12020 rc = VBOXSTRICTRC_VAL(rc2);
12021 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
12022 /** @todo We have to set pending-debug exceptions here when the guest is
12023 * single-stepping depending on the instruction that was interpreted. */
12024 Log4(("#GP rc=%Rrc\n", rc));
12025 break;
12026 }
12027 }
12028 }
12029 else
12030 rc = VERR_EM_INTERPRETER;
12031
12032 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
12033 ("#GP Unexpected rc=%Rrc\n", rc));
12034 return rc;
12035}
12036
12037
12038#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12039/**
12040 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
12041 * the exception reported in the VMX transient structure back into the VM.
12042 *
12043 * @remarks Requires uExitIntInfo in the VMX transient structure to be
12044 * up-to-date.
12045 */
12046static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12047{
12048 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12049
12050 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12051 hmR0VmxCheckExitDueToEventDelivery(). */
12052 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12053 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12054 AssertRCReturn(rc, rc);
12055 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12056
12057#ifdef DEBUG_ramshankar
12058 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12059 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12060 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12061#endif
12062
12063 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12064 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12065 return VINF_SUCCESS;
12066}
12067#endif
12068
12069
12070/**
12071 * VM-exit exception handler for #PF (Page-fault exception).
12072 */
12073static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12074{
12075 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12076 PVM pVM = pVCpu->CTX_SUFF(pVM);
12077 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12078 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12079 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12080 AssertRCReturn(rc, rc);
12081
12082#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12083 if (pVM->hm.s.fNestedPaging)
12084 {
12085 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12086 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12087 {
12088 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12089 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12090 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12091 }
12092 else
12093 {
12094 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12095 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12096 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12097 }
12098 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12099 return rc;
12100 }
12101#else
12102 Assert(!pVM->hm.s.fNestedPaging);
12103 NOREF(pVM);
12104#endif
12105
12106 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12107 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12108 if (pVmxTransient->fVectoringPF)
12109 {
12110 Assert(pVCpu->hm.s.Event.fPending);
12111 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12112 }
12113
12114 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12115 AssertRCReturn(rc, rc);
12116
12117 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12118 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12119
12120 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12121 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12122 (RTGCPTR)pVmxTransient->uExitQualification);
12123
12124 Log4(("#PF: rc=%Rrc\n", rc));
12125 if (rc == VINF_SUCCESS)
12126 {
12127 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12128 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12129 * memory? We don't update the whole state here... */
12130 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12131 | HM_CHANGED_GUEST_RSP
12132 | HM_CHANGED_GUEST_RFLAGS
12133 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12134 TRPMResetTrap(pVCpu);
12135 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12136 return rc;
12137 }
12138
12139 if (rc == VINF_EM_RAW_GUEST_TRAP)
12140 {
12141 if (!pVmxTransient->fVectoringDoublePF)
12142 {
12143 /* It's a guest page fault and needs to be reflected to the guest. */
12144 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12145 TRPMResetTrap(pVCpu);
12146 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12147 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12148 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12149 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12150 }
12151 else
12152 {
12153 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12154 TRPMResetTrap(pVCpu);
12155 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12156 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12157 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12158 }
12159
12160 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12161 return VINF_SUCCESS;
12162 }
12163
12164 TRPMResetTrap(pVCpu);
12165 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12166 return rc;
12167}
12168
12169/** @} */
12170
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette