VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 53628

Last change on this file since 53628 was 53628, checked in by vboxsync, 10 years ago

no else after break
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 505.3 KB
Line 
1/* $Id: HMVMXR0.cpp 53628 2015-01-01 21:15:33Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2014 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
41# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
43# define HMVMX_ALWAYS_CHECK_GUEST_STATE
44# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
45# define HMVMX_ALWAYS_TRAP_PF
46# define HMVMX_ALWAYS_SWAP_FPU_STATE
47# define HMVMX_ALWAYS_FLUSH_TLB
48# define HMVMX_ALWAYS_SWAP_EFER
49#endif
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define HMVMX_IS_64BIT_HOST_MODE() (true)
57typedef RTHCUINTREG HMVMXHCUINTREG;
58#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
59extern "C" uint32_t g_fVMXIs64bitHost;
60# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
61typedef uint64_t HMVMXHCUINTREG;
62#else
63# define HMVMX_IS_64BIT_HOST_MODE() (false)
64typedef RTHCUINTREG HMVMXHCUINTREG;
65#endif
66
67/** Use the function table. */
68#define HMVMX_USE_FUNCTION_TABLE
69
70/** Determine which tagged-TLB flush handler to use. */
71#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
72#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
73#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
74#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
75
76/** @name Updated-guest-state flags.
77 * @{ */
78#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
79#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
80#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
81#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
82#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
83#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
84#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
85#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
86#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
87#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
88#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
89#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
90#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
91#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
92#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
93#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
94#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
95#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
96#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
97#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
98#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
99 | HMVMX_UPDATED_GUEST_RSP \
100 | HMVMX_UPDATED_GUEST_RFLAGS \
101 | HMVMX_UPDATED_GUEST_CR0 \
102 | HMVMX_UPDATED_GUEST_CR3 \
103 | HMVMX_UPDATED_GUEST_CR4 \
104 | HMVMX_UPDATED_GUEST_GDTR \
105 | HMVMX_UPDATED_GUEST_IDTR \
106 | HMVMX_UPDATED_GUEST_LDTR \
107 | HMVMX_UPDATED_GUEST_TR \
108 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
109 | HMVMX_UPDATED_GUEST_DEBUG \
110 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
111 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
112 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
113 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
114 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
115 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
116 | HMVMX_UPDATED_GUEST_INTR_STATE \
117 | HMVMX_UPDATED_GUEST_APIC_STATE)
118/** @} */
119
120/** @name
121 * Flags to skip redundant reads of some common VMCS fields that are not part of
122 * the guest-CPU state but are in the transient structure.
123 */
124#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
125#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
131/** @} */
132
133/** @name
134 * States of the VMCS.
135 *
136 * This does not reflect all possible VMCS states but currently only those
137 * needed for maintaining the VMCS consistently even when thread-context hooks
138 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
139 */
140#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
141#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
142#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
143/** @} */
144
145/**
146 * Exception bitmap mask for real-mode guests (real-on-v86).
147 *
148 * We need to intercept all exceptions manually (except #PF). #NM is also
149 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
150 * even in real-mode if we have Nested Paging support.
151 */
152#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
153 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
154 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
155 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
156 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
157 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
158 | RT_BIT(X86_XCPT_XF))
159
160/**
161 * Exception bitmap mask for all contributory exceptions.
162 *
163 * Page fault is deliberately excluded here as it's conditional as to whether
164 * it's contributory or benign. Page faults are handled separately.
165 */
166#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
167 | RT_BIT(X86_XCPT_DE))
168
169/** Maximum VM-instruction error number. */
170#define HMVMX_INSTR_ERROR_MAX 28
171
172/** Profiling macro. */
173#ifdef HM_PROFILE_EXIT_DISPATCH
174# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
176#else
177# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
178# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
179#endif
180
181/** Assert that preemption is disabled or covered by thread-context hooks. */
182#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
184
185/** Assert that we haven't migrated CPUs when thread-context hooks are not
186 * used. */
187#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
188 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
189 ("Illegal migration! Entered on CPU %u Current %u\n", \
190 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
191
192/** Helper macro for VM-exit handlers called unexpectedly. */
193#define HMVMX_RETURN_UNEXPECTED_EXIT() \
194 do { \
195 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
196 return VERR_VMX_UNEXPECTED_EXIT; \
197 } while (0)
198
199
200/*******************************************************************************
201* Structures and Typedefs *
202*******************************************************************************/
203/**
204 * VMX transient state.
205 *
206 * A state structure for holding miscellaneous information across
207 * VMX non-root operation and restored after the transition.
208 */
209typedef struct VMXTRANSIENT
210{
211 /** The host's rflags/eflags. */
212 RTCCUINTREG uEflags;
213#if HC_ARCH_BITS == 32
214 uint32_t u32Alignment0;
215#endif
216 /** The guest's TPR value used for TPR shadowing. */
217 uint8_t u8GuestTpr;
218 /** Alignment. */
219 uint8_t abAlignment0[7];
220
221 /** The basic VM-exit reason. */
222 uint16_t uExitReason;
223 /** Alignment. */
224 uint16_t u16Alignment0;
225 /** The VM-exit interruption error code. */
226 uint32_t uExitIntErrorCode;
227 /** The VM-exit exit code qualification. */
228 uint64_t uExitQualification;
229
230 /** The VM-exit interruption-information field. */
231 uint32_t uExitIntInfo;
232 /** The VM-exit instruction-length field. */
233 uint32_t cbInstr;
234 /** The VM-exit instruction-information field. */
235 union
236 {
237 /** Plain unsigned int representation. */
238 uint32_t u;
239 /** INS and OUTS information. */
240 struct
241 {
242 uint32_t u6Reserved0 : 7;
243 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
244 uint32_t u3AddrSize : 3;
245 uint32_t u5Reserved1 : 5;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 uint32_t uReserved2 : 14;
249 } StrIo;
250 } ExitInstrInfo;
251 /** Whether the VM-entry failed or not. */
252 bool fVMEntryFailed;
253 /** Alignment. */
254 uint8_t abAlignment1[3];
255
256 /** The VM-entry interruption-information field. */
257 uint32_t uEntryIntInfo;
258 /** The VM-entry exception error code field. */
259 uint32_t uEntryXcptErrorCode;
260 /** The VM-entry instruction length field. */
261 uint32_t cbEntryInstr;
262
263 /** IDT-vectoring information field. */
264 uint32_t uIdtVectoringInfo;
265 /** IDT-vectoring error code. */
266 uint32_t uIdtVectoringErrorCode;
267
268 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
269 uint32_t fVmcsFieldsRead;
270
271 /** Whether the guest FPU was active at the time of VM-exit. */
272 bool fWasGuestFPUStateActive;
273 /** Whether the guest debug state was active at the time of VM-exit. */
274 bool fWasGuestDebugStateActive;
275 /** Whether the hyper debug state was active at the time of VM-exit. */
276 bool fWasHyperDebugStateActive;
277 /** Whether TSC-offsetting should be setup before VM-entry. */
278 bool fUpdateTscOffsettingAndPreemptTimer;
279 /** Whether the VM-exit was caused by a page-fault during delivery of a
280 * contributory exception or a page-fault. */
281 bool fVectoringDoublePF;
282 /** Whether the VM-exit was caused by a page-fault during delivery of an
283 * external interrupt or NMI. */
284 bool fVectoringPF;
285} VMXTRANSIENT;
286AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
287AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
288AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
289AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
290AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291/** Pointer to VMX transient state. */
292typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294
295/**
296 * MSR-bitmap read permissions.
297 */
298typedef enum VMXMSREXITREAD
299{
300 /** Reading this MSR causes a VM-exit. */
301 VMXMSREXIT_INTERCEPT_READ = 0xb,
302 /** Reading this MSR does not cause a VM-exit. */
303 VMXMSREXIT_PASSTHRU_READ
304} VMXMSREXITREAD;
305/** Pointer to MSR-bitmap read permissions. */
306typedef VMXMSREXITREAD* PVMXMSREXITREAD;
307
308/**
309 * MSR-bitmap write permissions.
310 */
311typedef enum VMXMSREXITWRITE
312{
313 /** Writing to this MSR causes a VM-exit. */
314 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
315 /** Writing to this MSR does not cause a VM-exit. */
316 VMXMSREXIT_PASSTHRU_WRITE
317} VMXMSREXITWRITE;
318/** Pointer to MSR-bitmap write permissions. */
319typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
320
321
322/**
323 * VMX VM-exit handler.
324 *
325 * @returns VBox status code.
326 * @param pVCpu Pointer to the VMCPU.
327 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
328 * out-of-sync. Make sure to update the required
329 * fields before using them.
330 * @param pVmxTransient Pointer to the VMX-transient structure.
331 */
332#ifndef HMVMX_USE_FUNCTION_TABLE
333typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
334#else
335typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
336/** Pointer to VM-exit handler. */
337typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
338#endif
339
340
341/*******************************************************************************
342* Internal Functions *
343*******************************************************************************/
344static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
345static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
346static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
347 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
348 bool fStepping, uint32_t *puIntState);
349#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
350static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
351#endif
352#ifndef HMVMX_USE_FUNCTION_TABLE
353DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
354# define HMVMX_EXIT_DECL static int
355#else
356# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
357#endif
358DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
359 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
360
361/** @name VM-exit handlers.
362 * @{
363 */
364static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
365static FNVMXEXITHANDLER hmR0VmxExitExtInt;
366static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
367static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
368static FNVMXEXITHANDLER hmR0VmxExitSipi;
369static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
370static FNVMXEXITHANDLER hmR0VmxExitSmi;
371static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
372static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
373static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
374static FNVMXEXITHANDLER hmR0VmxExitCpuid;
375static FNVMXEXITHANDLER hmR0VmxExitGetsec;
376static FNVMXEXITHANDLER hmR0VmxExitHlt;
377static FNVMXEXITHANDLER hmR0VmxExitInvd;
378static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
379static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
380static FNVMXEXITHANDLER hmR0VmxExitVmcall;
381static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
382static FNVMXEXITHANDLER hmR0VmxExitRsm;
383static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
384static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
385static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
386static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
387static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
388static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
389static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
390static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
391static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
392static FNVMXEXITHANDLER hmR0VmxExitMwait;
393static FNVMXEXITHANDLER hmR0VmxExitMtf;
394static FNVMXEXITHANDLER hmR0VmxExitMonitor;
395static FNVMXEXITHANDLER hmR0VmxExitPause;
396static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
397static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
398static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
399static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
400static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
401static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
402static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
403static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
404static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
405static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
406static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
407static FNVMXEXITHANDLER hmR0VmxExitRdrand;
408static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
409/** @} */
410
411static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
412static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
413static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
414static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
415static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
417#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
418static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
419#endif
420static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
421
422/*******************************************************************************
423* Global Variables *
424*******************************************************************************/
425#ifdef HMVMX_USE_FUNCTION_TABLE
426
427/**
428 * VMX_EXIT dispatch table.
429 */
430static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
431{
432 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
433 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
434 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
435 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
436 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
437 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
438 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
439 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
440 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
441 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
442 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
443 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
444 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
445 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
446 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
447 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
448 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
449 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
450 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
451 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
452 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
453 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
454 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
455 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
456 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
457 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
458 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
459 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
460 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
461 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
462 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
463 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
464 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
465 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
466 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
467 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
468 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
469 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
470 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
471 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
472 /* 40 UNDEFINED */ hmR0VmxExitPause,
473 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
474 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
475 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
476 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
477 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
478 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
479 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
480 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
481 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
482 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
483 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
484 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
485 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
486 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
487 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
488 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
490 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
491 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
492};
493#endif /* HMVMX_USE_FUNCTION_TABLE */
494
495#ifdef VBOX_STRICT
496static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
497{
498 /* 0 */ "(Not Used)",
499 /* 1 */ "VMCALL executed in VMX root operation.",
500 /* 2 */ "VMCLEAR with invalid physical address.",
501 /* 3 */ "VMCLEAR with VMXON pointer.",
502 /* 4 */ "VMLAUNCH with non-clear VMCS.",
503 /* 5 */ "VMRESUME with non-launched VMCS.",
504 /* 6 */ "VMRESUME after VMXOFF",
505 /* 7 */ "VM-entry with invalid control fields.",
506 /* 8 */ "VM-entry with invalid host state fields.",
507 /* 9 */ "VMPTRLD with invalid physical address.",
508 /* 10 */ "VMPTRLD with VMXON pointer.",
509 /* 11 */ "VMPTRLD with incorrect revision identifier.",
510 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
511 /* 13 */ "VMWRITE to read-only VMCS component.",
512 /* 14 */ "(Not Used)",
513 /* 15 */ "VMXON executed in VMX root operation.",
514 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
515 /* 17 */ "VM-entry with non-launched executing VMCS.",
516 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
517 /* 19 */ "VMCALL with non-clear VMCS.",
518 /* 20 */ "VMCALL with invalid VM-exit control fields.",
519 /* 21 */ "(Not Used)",
520 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
521 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
522 /* 24 */ "VMCALL with invalid SMM-monitor features.",
523 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
524 /* 26 */ "VM-entry with events blocked by MOV SS.",
525 /* 27 */ "(Not Used)",
526 /* 28 */ "Invalid operand to INVEPT/INVVPID."
527};
528#endif /* VBOX_STRICT */
529
530
531
532/**
533 * Updates the VM's last error record. If there was a VMX instruction error,
534 * reads the error data from the VMCS and updates VCPU's last error record as
535 * well.
536 *
537 * @param pVM Pointer to the VM.
538 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
539 * VERR_VMX_UNABLE_TO_START_VM or
540 * VERR_VMX_INVALID_VMCS_FIELD).
541 * @param rc The error code.
542 */
543static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
544{
545 AssertPtr(pVM);
546 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
547 || rc == VERR_VMX_UNABLE_TO_START_VM)
548 {
549 AssertPtrReturnVoid(pVCpu);
550 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
551 }
552 pVM->hm.s.lLastError = rc;
553}
554
555
556/**
557 * Reads the VM-entry interruption-information field from the VMCS into the VMX
558 * transient structure.
559 *
560 * @returns VBox status code.
561 * @param pVmxTransient Pointer to the VMX transient structure.
562 *
563 * @remarks No-long-jump zone!!!
564 */
565DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
566{
567 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
568 AssertRCReturn(rc, rc);
569 return VINF_SUCCESS;
570}
571
572
573/**
574 * Reads the VM-entry exception error code field from the VMCS into
575 * the VMX transient structure.
576 *
577 * @returns VBox status code.
578 * @param pVmxTransient Pointer to the VMX transient structure.
579 *
580 * @remarks No-long-jump zone!!!
581 */
582DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
583{
584 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
585 AssertRCReturn(rc, rc);
586 return VINF_SUCCESS;
587}
588
589
590/**
591 * Reads the VM-entry exception error code field from the VMCS into
592 * the VMX transient structure.
593 *
594 * @returns VBox status code.
595 * @param pVmxTransient Pointer to the VMX transient structure.
596 *
597 * @remarks No-long-jump zone!!!
598 */
599DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
600{
601 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
602 AssertRCReturn(rc, rc);
603 return VINF_SUCCESS;
604}
605
606
607/**
608 * Reads the VM-exit interruption-information field from the VMCS into the VMX
609 * transient structure.
610 *
611 * @returns VBox status code.
612 * @param pVmxTransient Pointer to the VMX transient structure.
613 */
614DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
615{
616 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
617 {
618 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
619 AssertRCReturn(rc, rc);
620 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
621 }
622 return VINF_SUCCESS;
623}
624
625
626/**
627 * Reads the VM-exit interruption error code from the VMCS into the VMX
628 * transient structure.
629 *
630 * @returns VBox status code.
631 * @param pVmxTransient Pointer to the VMX transient structure.
632 */
633DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
634{
635 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
636 {
637 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
638 AssertRCReturn(rc, rc);
639 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
640 }
641 return VINF_SUCCESS;
642}
643
644
645/**
646 * Reads the VM-exit instruction length field from the VMCS into the VMX
647 * transient structure.
648 *
649 * @returns VBox status code.
650 * @param pVCpu Pointer to the VMCPU.
651 * @param pVmxTransient Pointer to the VMX transient structure.
652 */
653DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
654{
655 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
656 {
657 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
658 AssertRCReturn(rc, rc);
659 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
660 }
661 return VINF_SUCCESS;
662}
663
664
665/**
666 * Reads the VM-exit instruction-information field from the VMCS into
667 * the VMX transient structure.
668 *
669 * @returns VBox status code.
670 * @param pVmxTransient Pointer to the VMX transient structure.
671 */
672DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
673{
674 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
675 {
676 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
677 AssertRCReturn(rc, rc);
678 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
679 }
680 return VINF_SUCCESS;
681}
682
683
684/**
685 * Reads the exit code qualification from the VMCS into the VMX transient
686 * structure.
687 *
688 * @returns VBox status code.
689 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
690 * case).
691 * @param pVmxTransient Pointer to the VMX transient structure.
692 */
693DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
694{
695 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
696 {
697 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
698 AssertRCReturn(rc, rc);
699 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
700 }
701 return VINF_SUCCESS;
702}
703
704
705/**
706 * Reads the IDT-vectoring information field from the VMCS into the VMX
707 * transient structure.
708 *
709 * @returns VBox status code.
710 * @param pVmxTransient Pointer to the VMX transient structure.
711 *
712 * @remarks No-long-jump zone!!!
713 */
714DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
715{
716 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
717 {
718 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
719 AssertRCReturn(rc, rc);
720 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
721 }
722 return VINF_SUCCESS;
723}
724
725
726/**
727 * Reads the IDT-vectoring error code from the VMCS into the VMX
728 * transient structure.
729 *
730 * @returns VBox status code.
731 * @param pVmxTransient Pointer to the VMX transient structure.
732 */
733DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
734{
735 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
736 {
737 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
738 AssertRCReturn(rc, rc);
739 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
740 }
741 return VINF_SUCCESS;
742}
743
744
745/**
746 * Enters VMX root mode operation on the current CPU.
747 *
748 * @returns VBox status code.
749 * @param pVM Pointer to the VM (optional, can be NULL, after
750 * a resume).
751 * @param HCPhysCpuPage Physical address of the VMXON region.
752 * @param pvCpuPage Pointer to the VMXON region.
753 */
754static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
755{
756 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
757 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
758 Assert(pvCpuPage);
759 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
760
761 if (pVM)
762 {
763 /* Write the VMCS revision dword to the VMXON region. */
764 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
765 }
766
767 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
768 RTCCUINTREG uEflags = ASMIntDisableFlags();
769
770 /* Enable the VMX bit in CR4 if necessary. */
771 RTCCUINTREG uCr4 = ASMGetCR4();
772 if (!(uCr4 & X86_CR4_VMXE))
773 ASMSetCR4(uCr4 | X86_CR4_VMXE);
774
775 /* Enter VMX root mode. */
776 int rc = VMXEnable(HCPhysCpuPage);
777 if (RT_FAILURE(rc))
778 ASMSetCR4(uCr4);
779
780 /* Restore interrupts. */
781 ASMSetFlags(uEflags);
782 return rc;
783}
784
785
786/**
787 * Exits VMX root mode operation on the current CPU.
788 *
789 * @returns VBox status code.
790 */
791static int hmR0VmxLeaveRootMode(void)
792{
793 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
794
795 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
796 RTCCUINTREG uEflags = ASMIntDisableFlags();
797
798 /* If we're for some reason not in VMX root mode, then don't leave it. */
799 RTCCUINTREG uHostCR4 = ASMGetCR4();
800
801 int rc;
802 if (uHostCR4 & X86_CR4_VMXE)
803 {
804 /* Exit VMX root mode and clear the VMX bit in CR4. */
805 VMXDisable();
806 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
807 rc = VINF_SUCCESS;
808 }
809 else
810 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
811
812 /* Restore interrupts. */
813 ASMSetFlags(uEflags);
814 return rc;
815}
816
817
818/**
819 * Allocates and maps one physically contiguous page. The allocated page is
820 * zero'd out. (Used by various VT-x structures).
821 *
822 * @returns IPRT status code.
823 * @param pMemObj Pointer to the ring-0 memory object.
824 * @param ppVirt Where to store the virtual address of the
825 * allocation.
826 * @param pPhys Where to store the physical address of the
827 * allocation.
828 */
829DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
830{
831 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
832 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
833 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
834
835 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
836 if (RT_FAILURE(rc))
837 return rc;
838 *ppVirt = RTR0MemObjAddress(*pMemObj);
839 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
840 ASMMemZero32(*ppVirt, PAGE_SIZE);
841 return VINF_SUCCESS;
842}
843
844
845/**
846 * Frees and unmaps an allocated physical page.
847 *
848 * @param pMemObj Pointer to the ring-0 memory object.
849 * @param ppVirt Where to re-initialize the virtual address of
850 * allocation as 0.
851 * @param pHCPhys Where to re-initialize the physical address of the
852 * allocation as 0.
853 */
854DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
855{
856 AssertPtr(pMemObj);
857 AssertPtr(ppVirt);
858 AssertPtr(pHCPhys);
859 if (*pMemObj != NIL_RTR0MEMOBJ)
860 {
861 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
862 AssertRC(rc);
863 *pMemObj = NIL_RTR0MEMOBJ;
864 *ppVirt = 0;
865 *pHCPhys = 0;
866 }
867}
868
869
870/**
871 * Worker function to free VT-x related structures.
872 *
873 * @returns IPRT status code.
874 * @param pVM Pointer to the VM.
875 */
876static void hmR0VmxStructsFree(PVM pVM)
877{
878 for (VMCPUID i = 0; i < pVM->cCpus; i++)
879 {
880 PVMCPU pVCpu = &pVM->aCpus[i];
881 AssertPtr(pVCpu);
882
883 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
884 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
885
886 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
887 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
888
889 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
890 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
891 }
892
893 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
894#ifdef VBOX_WITH_CRASHDUMP_MAGIC
895 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
896#endif
897}
898
899
900/**
901 * Worker function to allocate VT-x related VM structures.
902 *
903 * @returns IPRT status code.
904 * @param pVM Pointer to the VM.
905 */
906static int hmR0VmxStructsAlloc(PVM pVM)
907{
908 /*
909 * Initialize members up-front so we can cleanup properly on allocation failure.
910 */
911#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
912 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
913 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
914 pVM->hm.s.vmx.HCPhys##a_Name = 0;
915
916#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
917 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
918 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
919 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
920
921#ifdef VBOX_WITH_CRASHDUMP_MAGIC
922 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
923#endif
924 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
925
926 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
927 for (VMCPUID i = 0; i < pVM->cCpus; i++)
928 {
929 PVMCPU pVCpu = &pVM->aCpus[i];
930 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
931 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
932 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
935 }
936#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
937#undef VMXLOCAL_INIT_VM_MEMOBJ
938
939 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
940 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
941 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
942 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
943
944 /*
945 * Allocate all the VT-x structures.
946 */
947 int rc = VINF_SUCCESS;
948#ifdef VBOX_WITH_CRASHDUMP_MAGIC
949 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
950 if (RT_FAILURE(rc))
951 goto cleanup;
952 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
953 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
954#endif
955
956 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
957 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
958 {
959 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
960 &pVM->hm.s.vmx.HCPhysApicAccess);
961 if (RT_FAILURE(rc))
962 goto cleanup;
963 }
964
965 /*
966 * Initialize per-VCPU VT-x structures.
967 */
968 for (VMCPUID i = 0; i < pVM->cCpus; i++)
969 {
970 PVMCPU pVCpu = &pVM->aCpus[i];
971 AssertPtr(pVCpu);
972
973 /* Allocate the VM control structure (VMCS). */
974 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
975 if (RT_FAILURE(rc))
976 goto cleanup;
977
978 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
979 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
980 {
981 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
982 &pVCpu->hm.s.vmx.HCPhysVirtApic);
983 if (RT_FAILURE(rc))
984 goto cleanup;
985 }
986
987 /*
988 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
989 * transparent accesses of specific MSRs.
990 *
991 * If the condition for enabling MSR bitmaps changes here, don't forget to
992 * update HMIsMsrBitmapsAvailable().
993 */
994 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
995 {
996 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
997 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
998 if (RT_FAILURE(rc))
999 goto cleanup;
1000 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1001 }
1002
1003 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1009 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1010 if (RT_FAILURE(rc))
1011 goto cleanup;
1012 }
1013
1014 return VINF_SUCCESS;
1015
1016cleanup:
1017 hmR0VmxStructsFree(pVM);
1018 return rc;
1019}
1020
1021
1022/**
1023 * Does global VT-x initialization (called during module initialization).
1024 *
1025 * @returns VBox status code.
1026 */
1027VMMR0DECL(int) VMXR0GlobalInit(void)
1028{
1029#ifdef HMVMX_USE_FUNCTION_TABLE
1030 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1031# ifdef VBOX_STRICT
1032 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1033 Assert(g_apfnVMExitHandlers[i]);
1034# endif
1035#endif
1036 return VINF_SUCCESS;
1037}
1038
1039
1040/**
1041 * Does global VT-x termination (called during module termination).
1042 */
1043VMMR0DECL(void) VMXR0GlobalTerm()
1044{
1045 /* Nothing to do currently. */
1046}
1047
1048
1049/**
1050 * Sets up and activates VT-x on the current CPU.
1051 *
1052 * @returns VBox status code.
1053 * @param pCpu Pointer to the global CPU info struct.
1054 * @param pVM Pointer to the VM (can be NULL after a host resume
1055 * operation).
1056 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1057 * fEnabledByHost is true).
1058 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1059 * @a fEnabledByHost is true).
1060 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1061 * enable VT-x on the host.
1062 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1063 */
1064VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1065 void *pvMsrs)
1066{
1067 Assert(pCpu);
1068 Assert(pvMsrs);
1069 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1070
1071 /* Enable VT-x if it's not already enabled by the host. */
1072 if (!fEnabledByHost)
1073 {
1074 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1075 if (RT_FAILURE(rc))
1076 return rc;
1077 }
1078
1079 /*
1080 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1081 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1082 */
1083 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1084 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1085 {
1086 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1087 pCpu->fFlushAsidBeforeUse = false;
1088 }
1089 else
1090 pCpu->fFlushAsidBeforeUse = true;
1091
1092 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1093 ++pCpu->cTlbFlushes;
1094
1095 return VINF_SUCCESS;
1096}
1097
1098
1099/**
1100 * Deactivates VT-x on the current CPU.
1101 *
1102 * @returns VBox status code.
1103 * @param pCpu Pointer to the global CPU info struct.
1104 * @param pvCpuPage Pointer to the VMXON region.
1105 * @param HCPhysCpuPage Physical address of the VMXON region.
1106 *
1107 * @remarks This function should never be called when SUPR0EnableVTx() or
1108 * similar was used to enable VT-x on the host.
1109 */
1110VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1111{
1112 NOREF(pCpu);
1113 NOREF(pvCpuPage);
1114 NOREF(HCPhysCpuPage);
1115
1116 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1117 return hmR0VmxLeaveRootMode();
1118}
1119
1120
1121/**
1122 * Sets the permission bits for the specified MSR in the MSR bitmap.
1123 *
1124 * @param pVCpu Pointer to the VMCPU.
1125 * @param uMSR The MSR value.
1126 * @param enmRead Whether reading this MSR causes a VM-exit.
1127 * @param enmWrite Whether writing this MSR causes a VM-exit.
1128 */
1129static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1130{
1131 int32_t iBit;
1132 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1133
1134 /*
1135 * Layout:
1136 * 0x000 - 0x3ff - Low MSR read bits
1137 * 0x400 - 0x7ff - High MSR read bits
1138 * 0x800 - 0xbff - Low MSR write bits
1139 * 0xc00 - 0xfff - High MSR write bits
1140 */
1141 if (uMsr <= 0x00001FFF)
1142 iBit = uMsr;
1143 else if ( uMsr >= 0xC0000000
1144 && uMsr <= 0xC0001FFF)
1145 {
1146 iBit = (uMsr - 0xC0000000);
1147 pbMsrBitmap += 0x400;
1148 }
1149 else
1150 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1151
1152 Assert(iBit <= 0x1fff);
1153 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1154 ASMBitSet(pbMsrBitmap, iBit);
1155 else
1156 ASMBitClear(pbMsrBitmap, iBit);
1157
1158 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1159 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1160 else
1161 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1162}
1163
1164
1165#ifdef VBOX_STRICT
1166/**
1167 * Gets the permission bits for the specified MSR in the MSR bitmap.
1168 *
1169 * @returns VBox status code.
1170 * @retval VINF_SUCCESS if the specified MSR is found.
1171 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1172 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1173 *
1174 * @param pVCpu Pointer to the VMCPU.
1175 * @param uMsr The MSR.
1176 * @param penmRead Where to store the read permissions.
1177 * @param penmWrite Where to store the write permissions.
1178 */
1179static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1180{
1181 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1182 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1183 int32_t iBit;
1184 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1185
1186 /* See hmR0VmxSetMsrPermission() for the layout. */
1187 if (uMsr <= 0x00001FFF)
1188 iBit = uMsr;
1189 else if ( uMsr >= 0xC0000000
1190 && uMsr <= 0xC0001FFF)
1191 {
1192 iBit = (uMsr - 0xC0000000);
1193 pbMsrBitmap += 0x400;
1194 }
1195 else
1196 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1197
1198 Assert(iBit <= 0x1fff);
1199 if (ASMBitTest(pbMsrBitmap, iBit))
1200 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1201 else
1202 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1203
1204 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1205 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1206 else
1207 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1208 return VINF_SUCCESS;
1209}
1210#endif /* VBOX_STRICT */
1211
1212
1213/**
1214 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1215 * area.
1216 *
1217 * @returns VBox status code.
1218 * @param pVCpu Pointer to the VMCPU.
1219 * @param cMsrs The number of MSRs.
1220 */
1221DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1222{
1223 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1224 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1225 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1226 {
1227 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1228 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1229 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1230 }
1231
1232 /* Update number of guest MSRs to load/store across the world-switch. */
1233 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1234 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1235
1236 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1237 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1238
1239 /* Update the VCPU's copy of the MSR count. */
1240 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1241
1242 return VINF_SUCCESS;
1243}
1244
1245
1246/**
1247 * Adds a new (or updates the value of an existing) guest/host MSR
1248 * pair to be swapped during the world-switch as part of the
1249 * auto-load/store MSR area in the VMCS.
1250 *
1251 * @returns true if the MSR was added -and- its value was updated, false
1252 * otherwise.
1253 * @param pVCpu Pointer to the VMCPU.
1254 * @param uMsr The MSR.
1255 * @param uGuestMsr Value of the guest MSR.
1256 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1257 * necessary.
1258 */
1259static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1260{
1261 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1262 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1263 uint32_t i;
1264 for (i = 0; i < cMsrs; i++)
1265 {
1266 if (pGuestMsr->u32Msr == uMsr)
1267 break;
1268 pGuestMsr++;
1269 }
1270
1271 bool fAdded = false;
1272 if (i == cMsrs)
1273 {
1274 ++cMsrs;
1275 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1276 AssertRC(rc);
1277
1278 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1279 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1280 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1281
1282 fAdded = true;
1283 }
1284
1285 /* Update the MSR values in the auto-load/store MSR area. */
1286 pGuestMsr->u32Msr = uMsr;
1287 pGuestMsr->u64Value = uGuestMsrValue;
1288
1289 /* Create/update the MSR slot in the host MSR area. */
1290 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1291 pHostMsr += i;
1292 pHostMsr->u32Msr = uMsr;
1293
1294 /*
1295 * Update the host MSR only when requested by the caller AND when we're
1296 * adding it to the auto-load/store area. Otherwise, it would have been
1297 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1298 */
1299 bool fUpdatedMsrValue = false;
1300 if ( fAdded
1301 && fUpdateHostMsr)
1302 {
1303 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1304 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1305 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1306 fUpdatedMsrValue = true;
1307 }
1308
1309 return fUpdatedMsrValue;
1310}
1311
1312
1313/**
1314 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1315 * auto-load/store MSR area in the VMCS.
1316 *
1317 * @returns VBox status code.
1318 * @param pVCpu Pointer to the VMCPU.
1319 * @param uMsr The MSR.
1320 */
1321static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1322{
1323 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1324 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1325 for (uint32_t i = 0; i < cMsrs; i++)
1326 {
1327 /* Find the MSR. */
1328 if (pGuestMsr->u32Msr == uMsr)
1329 {
1330 /* If it's the last MSR, simply reduce the count. */
1331 if (i == cMsrs - 1)
1332 {
1333 --cMsrs;
1334 break;
1335 }
1336
1337 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1338 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1339 pLastGuestMsr += cMsrs - 1;
1340 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1341 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1342
1343 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1344 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1345 pLastHostMsr += cMsrs - 1;
1346 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1347 pHostMsr->u64Value = pLastHostMsr->u64Value;
1348 --cMsrs;
1349 break;
1350 }
1351 pGuestMsr++;
1352 }
1353
1354 /* Update the VMCS if the count changed (meaning the MSR was found). */
1355 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1356 {
1357 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1358 AssertRCReturn(rc, rc);
1359
1360 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1361 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1362 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1363
1364 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1365 return VINF_SUCCESS;
1366 }
1367
1368 return VERR_NOT_FOUND;
1369}
1370
1371
1372/**
1373 * Checks if the specified guest MSR is part of the auto-load/store area in
1374 * the VMCS.
1375 *
1376 * @returns true if found, false otherwise.
1377 * @param pVCpu Pointer to the VMCPU.
1378 * @param uMsr The MSR to find.
1379 */
1380static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1381{
1382 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1383 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1384
1385 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1386 {
1387 if (pGuestMsr->u32Msr == uMsr)
1388 return true;
1389 }
1390 return false;
1391}
1392
1393
1394/**
1395 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1396 *
1397 * @param pVCpu Pointer to the VMCPU.
1398 *
1399 * @remarks No-long-jump zone!!!
1400 */
1401static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1402{
1403 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1404 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1405 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1406 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1407
1408 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1409 {
1410 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1411
1412 /*
1413 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1414 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1415 */
1416 if (pHostMsr->u32Msr == MSR_K6_EFER)
1417 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1418 else
1419 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1420 }
1421
1422 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1423}
1424
1425
1426#if HC_ARCH_BITS == 64
1427/**
1428 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1429 * perform lazy restoration of the host MSRs while leaving VT-x.
1430 *
1431 * @param pVCpu Pointer to the VMCPU.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438
1439 /*
1440 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1441 */
1442 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1443 {
1444 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1445 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1446 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1447 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1448 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1449 }
1450}
1451
1452
1453/**
1454 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1455 * lazily while leaving VT-x.
1456 *
1457 * @returns true if it does, false otherwise.
1458 * @param pVCpu Pointer to the VMCPU.
1459 * @param uMsr The MSR to check.
1460 */
1461static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1462{
1463 NOREF(pVCpu);
1464 switch (uMsr)
1465 {
1466 case MSR_K8_LSTAR:
1467 case MSR_K6_STAR:
1468 case MSR_K8_SF_MASK:
1469 case MSR_K8_KERNEL_GS_BASE:
1470 return true;
1471 }
1472 return false;
1473}
1474
1475
1476/**
1477 * Saves a set of guest MSRs back into the guest-CPU context.
1478 *
1479 * @param pVCpu Pointer to the VMCPU.
1480 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1481 * out-of-sync. Make sure to update the required fields
1482 * before using them.
1483 *
1484 * @remarks No-long-jump zone!!!
1485 */
1486static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1487{
1488 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1489 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1490
1491 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1492 {
1493 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1494 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1495 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1496 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1497 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1498 }
1499}
1500
1501
1502/**
1503 * Loads a set of guests MSRs to allow read/passthru to the guest.
1504 *
1505 * The name of this function is slightly confusing. This function does NOT
1506 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1507 * common prefix for functions dealing with "lazy restoration" of the shared
1508 * MSRs.
1509 *
1510 * @param pVCpu Pointer to the VMCPU.
1511 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1512 * out-of-sync. Make sure to update the required fields
1513 * before using them.
1514 *
1515 * @remarks No-long-jump zone!!!
1516 */
1517static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1518{
1519 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1520 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1521
1522#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1523 do { \
1524 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1525 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1526 else \
1527 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1528 } while (0)
1529
1530 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1531 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1532 {
1533 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1534 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1535 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1536 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1537 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1538 }
1539 else
1540 {
1541 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1542 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1543 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1544 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1545 }
1546
1547#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1548}
1549
1550
1551/**
1552 * Performs lazy restoration of the set of host MSRs if they were previously
1553 * loaded with guest MSR values.
1554 *
1555 * @param pVCpu Pointer to the VMCPU.
1556 *
1557 * @remarks No-long-jump zone!!!
1558 * @remarks The guest MSRs should have been saved back into the guest-CPU
1559 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1560 */
1561static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1562{
1563 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1564 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1565
1566 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1567 {
1568 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1569 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1570 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1571 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1572 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1573 }
1574 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1575}
1576#endif /* HC_ARCH_BITS == 64 */
1577
1578
1579/**
1580 * Verifies that our cached values of the VMCS controls are all
1581 * consistent with what's actually present in the VMCS.
1582 *
1583 * @returns VBox status code.
1584 * @param pVCpu Pointer to the VMCPU.
1585 */
1586static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1587{
1588 uint32_t u32Val;
1589 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1590 AssertRCReturn(rc, rc);
1591 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1592 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1593
1594 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1595 AssertRCReturn(rc, rc);
1596 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1597 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1598
1599 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1600 AssertRCReturn(rc, rc);
1601 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1602 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1603
1604 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1605 AssertRCReturn(rc, rc);
1606 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1607 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1608
1609 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1610 {
1611 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1612 AssertRCReturn(rc, rc);
1613 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1614 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1615 }
1616
1617 return VINF_SUCCESS;
1618}
1619
1620
1621#ifdef VBOX_STRICT
1622/**
1623 * Verifies that our cached host EFER value has not changed
1624 * since we cached it.
1625 *
1626 * @param pVCpu Pointer to the VMCPU.
1627 */
1628static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1629{
1630 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1631
1632 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1633 {
1634 uint64_t u64Val;
1635 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1636 AssertRC(rc);
1637
1638 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1639 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1640 }
1641}
1642
1643
1644/**
1645 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1646 * VMCS are correct.
1647 *
1648 * @param pVCpu Pointer to the VMCPU.
1649 */
1650static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1651{
1652 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1653
1654 /* Verify MSR counts in the VMCS are what we think it should be. */
1655 uint32_t cMsrs;
1656 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1657 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1658
1659 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1660 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1661
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1663 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1664
1665 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1666 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1667 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1668 {
1669 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1670 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1671 pGuestMsr->u32Msr, cMsrs));
1672
1673 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1674 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1675 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1676
1677 /* Verify that the permissions are as expected in the MSR bitmap. */
1678 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1679 {
1680 VMXMSREXITREAD enmRead;
1681 VMXMSREXITWRITE enmWrite;
1682 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1683 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1684 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1685 {
1686 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1687 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1688 }
1689 else
1690 {
1691 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1692 pGuestMsr->u32Msr, cMsrs));
1693 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1694 pGuestMsr->u32Msr, cMsrs));
1695 }
1696 }
1697 }
1698}
1699#endif /* VBOX_STRICT */
1700
1701
1702/**
1703 * Flushes the TLB using EPT.
1704 *
1705 * @returns VBox status code.
1706 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1707 * enmFlush).
1708 * @param enmFlush Type of flush.
1709 *
1710 * @remarks Caller is responsible for making sure this function is called only
1711 * when NestedPaging is supported and providing @a enmFlush that is
1712 * supported by the CPU.
1713 * @remarks Can be called with interrupts disabled.
1714 */
1715static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1716{
1717 uint64_t au64Descriptor[2];
1718 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1719 au64Descriptor[0] = 0;
1720 else
1721 {
1722 Assert(pVCpu);
1723 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1724 }
1725 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1726
1727 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1728 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1729 rc));
1730 if ( RT_SUCCESS(rc)
1731 && pVCpu)
1732 {
1733 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1734 }
1735}
1736
1737
1738/**
1739 * Flushes the TLB using VPID.
1740 *
1741 * @returns VBox status code.
1742 * @param pVM Pointer to the VM.
1743 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1744 * enmFlush).
1745 * @param enmFlush Type of flush.
1746 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1747 * on @a enmFlush).
1748 *
1749 * @remarks Can be called with interrupts disabled.
1750 */
1751static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1752{
1753 NOREF(pVM);
1754 AssertPtr(pVM);
1755 Assert(pVM->hm.s.vmx.fVpid);
1756
1757 uint64_t au64Descriptor[2];
1758 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1759 {
1760 au64Descriptor[0] = 0;
1761 au64Descriptor[1] = 0;
1762 }
1763 else
1764 {
1765 AssertPtr(pVCpu);
1766 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1767 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1768 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1769 au64Descriptor[1] = GCPtr;
1770 }
1771
1772 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1773 AssertMsg(rc == VINF_SUCCESS,
1774 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1775 if ( RT_SUCCESS(rc)
1776 && pVCpu)
1777 {
1778 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1779 }
1780}
1781
1782
1783/**
1784 * Invalidates a guest page by guest virtual address. Only relevant for
1785 * EPT/VPID, otherwise there is nothing really to invalidate.
1786 *
1787 * @returns VBox status code.
1788 * @param pVM Pointer to the VM.
1789 * @param pVCpu Pointer to the VMCPU.
1790 * @param GCVirt Guest virtual address of the page to invalidate.
1791 */
1792VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1793{
1794 AssertPtr(pVM);
1795 AssertPtr(pVCpu);
1796 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1797
1798 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1799 if (!fFlushPending)
1800 {
1801 /*
1802 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1803 * See @bugref{6043} and @bugref{6177}.
1804 *
1805 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1806 * function maybe called in a loop with individual addresses.
1807 */
1808 if (pVM->hm.s.vmx.fVpid)
1809 {
1810 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1811 {
1812 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1813 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1814 }
1815 else
1816 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1817 }
1818 else if (pVM->hm.s.fNestedPaging)
1819 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1820 }
1821
1822 return VINF_SUCCESS;
1823}
1824
1825
1826/**
1827 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1828 * otherwise there is nothing really to invalidate.
1829 *
1830 * @returns VBox status code.
1831 * @param pVM Pointer to the VM.
1832 * @param pVCpu Pointer to the VMCPU.
1833 * @param GCPhys Guest physical address of the page to invalidate.
1834 */
1835VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1836{
1837 NOREF(pVM); NOREF(GCPhys);
1838 LogFlowFunc(("%RGp\n", GCPhys));
1839
1840 /*
1841 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1842 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1843 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1844 */
1845 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1846 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1847 return VINF_SUCCESS;
1848}
1849
1850
1851/**
1852 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1853 * case where neither EPT nor VPID is supported by the CPU.
1854 *
1855 * @param pVM Pointer to the VM.
1856 * @param pVCpu Pointer to the VMCPU.
1857 * @param pCpu Pointer to the global HM struct.
1858 *
1859 * @remarks Called with interrupts disabled.
1860 */
1861static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1862{
1863 AssertPtr(pVCpu);
1864 AssertPtr(pCpu);
1865 NOREF(pVM);
1866
1867 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1868
1869 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1870#if 0
1871 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1872 pVCpu->hm.s.TlbShootdown.cPages = 0;
1873#endif
1874
1875 Assert(pCpu->idCpu != NIL_RTCPUID);
1876 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1877 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1878 pVCpu->hm.s.fForceTLBFlush = false;
1879 return;
1880}
1881
1882
1883/**
1884 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1885 *
1886 * @param pVM Pointer to the VM.
1887 * @param pVCpu Pointer to the VMCPU.
1888 * @param pCpu Pointer to the global HM CPU struct.
1889 * @remarks All references to "ASID" in this function pertains to "VPID" in
1890 * Intel's nomenclature. The reason is, to avoid confusion in compare
1891 * statements since the host-CPU copies are named "ASID".
1892 *
1893 * @remarks Called with interrupts disabled.
1894 */
1895static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1896{
1897#ifdef VBOX_WITH_STATISTICS
1898 bool fTlbFlushed = false;
1899# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1900# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1901 if (!fTlbFlushed) \
1902 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1903 } while (0)
1904#else
1905# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1906# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1907#endif
1908
1909 AssertPtr(pVM);
1910 AssertPtr(pCpu);
1911 AssertPtr(pVCpu);
1912 Assert(pCpu->idCpu != NIL_RTCPUID);
1913
1914 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1915 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1916 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1917
1918 /*
1919 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1920 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1921 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1922 */
1923 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1924 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1925 {
1926 ++pCpu->uCurrentAsid;
1927 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1928 {
1929 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1930 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1931 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1932 }
1933
1934 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1935 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1936 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1937
1938 /*
1939 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1940 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1941 */
1942 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1943 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1944 HMVMX_SET_TAGGED_TLB_FLUSHED();
1945 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1946 }
1947
1948 /* Check for explicit TLB shootdowns. */
1949 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1950 {
1951 /*
1952 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1953 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1954 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1955 * but not guest-physical mappings.
1956 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1957 */
1958 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1959 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1960 HMVMX_SET_TAGGED_TLB_FLUSHED();
1961 }
1962
1963 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1964 * where it is commented out. Support individual entry flushing
1965 * someday. */
1966#if 0
1967 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1968 {
1969 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1970
1971 /*
1972 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1973 * as supported by the CPU.
1974 */
1975 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1976 {
1977 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1978 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1979 }
1980 else
1981 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1982
1983 HMVMX_SET_TAGGED_TLB_FLUSHED();
1984 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1985 pVCpu->hm.s.TlbShootdown.cPages = 0;
1986 }
1987#endif
1988
1989 pVCpu->hm.s.fForceTLBFlush = false;
1990
1991 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1992
1993 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1994 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1995 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1996 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1997 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1998 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1999 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2000 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2001 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2002
2003 /* Update VMCS with the VPID. */
2004 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2005 AssertRC(rc);
2006
2007#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2008}
2009
2010
2011/**
2012 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2013 *
2014 * @returns VBox status code.
2015 * @param pVM Pointer to the VM.
2016 * @param pVCpu Pointer to the VMCPU.
2017 * @param pCpu Pointer to the global HM CPU struct.
2018 *
2019 * @remarks Called with interrupts disabled.
2020 */
2021static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2022{
2023 AssertPtr(pVM);
2024 AssertPtr(pVCpu);
2025 AssertPtr(pCpu);
2026 Assert(pCpu->idCpu != NIL_RTCPUID);
2027 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2028 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2029
2030 /*
2031 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2032 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2033 */
2034 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2035 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2036 {
2037 pVCpu->hm.s.fForceTLBFlush = true;
2038 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2039 }
2040
2041 /* Check for explicit TLB shootdown flushes. */
2042 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2043 {
2044 pVCpu->hm.s.fForceTLBFlush = true;
2045 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2046 }
2047
2048 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2049 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2050
2051 if (pVCpu->hm.s.fForceTLBFlush)
2052 {
2053 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2054 pVCpu->hm.s.fForceTLBFlush = false;
2055 }
2056 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2057 * where it is commented out. Support individual entry flushing
2058 * someday. */
2059#if 0
2060 else
2061 {
2062 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2063 {
2064 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2065 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2066 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2067 }
2068 else
2069 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2070
2071 pVCpu->hm.s.TlbShootdown.cPages = 0;
2072 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2073 }
2074#endif
2075}
2076
2077
2078/**
2079 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2080 *
2081 * @returns VBox status code.
2082 * @param pVM Pointer to the VM.
2083 * @param pVCpu Pointer to the VMCPU.
2084 * @param pCpu Pointer to the global HM CPU struct.
2085 *
2086 * @remarks Called with interrupts disabled.
2087 */
2088static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089{
2090 AssertPtr(pVM);
2091 AssertPtr(pVCpu);
2092 AssertPtr(pCpu);
2093 Assert(pCpu->idCpu != NIL_RTCPUID);
2094 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2095 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2096
2097 /*
2098 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2099 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2100 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2101 */
2102 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104 {
2105 pVCpu->hm.s.fForceTLBFlush = true;
2106 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107 }
2108
2109 /* Check for explicit TLB shootdown flushes. */
2110 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111 {
2112 /*
2113 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2114 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2115 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2116 */
2117 pVCpu->hm.s.fForceTLBFlush = true;
2118 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2119 }
2120
2121 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2122 if (pVCpu->hm.s.fForceTLBFlush)
2123 {
2124 ++pCpu->uCurrentAsid;
2125 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2126 {
2127 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2128 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2129 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2130 }
2131
2132 pVCpu->hm.s.fForceTLBFlush = false;
2133 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2134 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2135 if (pCpu->fFlushAsidBeforeUse)
2136 {
2137 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2138 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2139 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2140 {
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2142 pCpu->fFlushAsidBeforeUse = false;
2143 }
2144 else
2145 {
2146 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2147 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2148 }
2149 }
2150 }
2151 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2152 * where it is commented out. Support individual entry flushing
2153 * someday. */
2154#if 0
2155 else
2156 {
2157 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2158 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2159 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2160 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2161
2162 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2163 {
2164 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2165 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2166 {
2167 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2168 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2169 }
2170 else
2171 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2172
2173 pVCpu->hm.s.TlbShootdown.cPages = 0;
2174 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2175 }
2176 else
2177 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2178 }
2179#endif
2180
2181 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2182 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2183 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2184 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2185 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2186 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2187 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2188
2189 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2190 AssertRC(rc);
2191}
2192
2193
2194/**
2195 * Flushes the guest TLB entry based on CPU capabilities.
2196 *
2197 * @param pVCpu Pointer to the VMCPU.
2198 * @param pCpu Pointer to the global HM CPU struct.
2199 */
2200DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2201{
2202#ifdef HMVMX_ALWAYS_FLUSH_TLB
2203 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2204#endif
2205 PVM pVM = pVCpu->CTX_SUFF(pVM);
2206 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2207 {
2208 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2209 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2210 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2211 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2212 default:
2213 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2214 break;
2215 }
2216
2217 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2218 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2219
2220 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2221}
2222
2223
2224/**
2225 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2226 * TLB entries from the host TLB before VM-entry.
2227 *
2228 * @returns VBox status code.
2229 * @param pVM Pointer to the VM.
2230 */
2231static int hmR0VmxSetupTaggedTlb(PVM pVM)
2232{
2233 /*
2234 * Determine optimal flush type for Nested Paging.
2235 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2236 * guest execution (see hmR3InitFinalizeR0()).
2237 */
2238 if (pVM->hm.s.fNestedPaging)
2239 {
2240 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2241 {
2242 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2243 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2244 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2245 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2246 else
2247 {
2248 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2249 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2250 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2251 }
2252
2253 /* Make sure the write-back cacheable memory type for EPT is supported. */
2254 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2255 {
2256 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2257 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2259 }
2260 }
2261 else
2262 {
2263 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2264 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2265 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2266 }
2267 }
2268
2269 /*
2270 * Determine optimal flush type for VPID.
2271 */
2272 if (pVM->hm.s.vmx.fVpid)
2273 {
2274 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2275 {
2276 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2277 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2278 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2279 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2280 else
2281 {
2282 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2283 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2284 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2285 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2286 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2287 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2288 pVM->hm.s.vmx.fVpid = false;
2289 }
2290 }
2291 else
2292 {
2293 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2294 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2295 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2296 pVM->hm.s.vmx.fVpid = false;
2297 }
2298 }
2299
2300 /*
2301 * Setup the handler for flushing tagged-TLBs.
2302 */
2303 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2304 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2305 else if (pVM->hm.s.fNestedPaging)
2306 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2307 else if (pVM->hm.s.vmx.fVpid)
2308 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2309 else
2310 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2311 return VINF_SUCCESS;
2312}
2313
2314
2315/**
2316 * Sets up pin-based VM-execution controls in the VMCS.
2317 *
2318 * @returns VBox status code.
2319 * @param pVM Pointer to the VM.
2320 * @param pVCpu Pointer to the VMCPU.
2321 */
2322static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2323{
2324 AssertPtr(pVM);
2325 AssertPtr(pVCpu);
2326
2327 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2328 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2329
2330 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2331 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2332
2333 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2334 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2335
2336 /* Enable the VMX preemption timer. */
2337 if (pVM->hm.s.vmx.fUsePreemptTimer)
2338 {
2339 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2340 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2341 }
2342
2343 if ((val & zap) != val)
2344 {
2345 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2346 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2347 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2348 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2349 }
2350
2351 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2352 AssertRCReturn(rc, rc);
2353
2354 pVCpu->hm.s.vmx.u32PinCtls = val;
2355 return rc;
2356}
2357
2358
2359/**
2360 * Sets up processor-based VM-execution controls in the VMCS.
2361 *
2362 * @returns VBox status code.
2363 * @param pVM Pointer to the VM.
2364 * @param pVMCPU Pointer to the VMCPU.
2365 */
2366static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2367{
2368 AssertPtr(pVM);
2369 AssertPtr(pVCpu);
2370
2371 int rc = VERR_INTERNAL_ERROR_5;
2372 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2373 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2374
2375 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2376 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2377 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2378 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2379 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2380 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2381 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2382
2383 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2384 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2385 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2386 {
2387 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2388 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2389 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2390 }
2391
2392 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2393 if (!pVM->hm.s.fNestedPaging)
2394 {
2395 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2396 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2397 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2398 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2399 }
2400
2401 /* Use TPR shadowing if supported by the CPU. */
2402 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2403 {
2404 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2405 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2406 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2407 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2408 AssertRCReturn(rc, rc);
2409
2410 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2411 /* CR8 writes cause a VM-exit based on TPR threshold. */
2412 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2413 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2414 }
2415 else
2416 {
2417 /*
2418 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2419 * Set this control only for 64-bit guests.
2420 */
2421 if (pVM->hm.s.fAllow64BitGuests)
2422 {
2423 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2424 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2425 }
2426 }
2427
2428 /* Use MSR-bitmaps if supported by the CPU. */
2429 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2430 {
2431 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2432
2433 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2434 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2435 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2436 AssertRCReturn(rc, rc);
2437
2438 /*
2439 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2440 * automatically using dedicated fields in the VMCS.
2441 */
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447
2448#if HC_ARCH_BITS == 64
2449 /*
2450 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2451 */
2452 if (pVM->hm.s.fAllow64BitGuests)
2453 {
2454 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2455 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2456 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2457 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2458 }
2459#endif
2460 }
2461
2462 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2463 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2464 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2465
2466 if ((val & zap) != val)
2467 {
2468 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2469 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2470 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2471 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2472 }
2473
2474 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2475 AssertRCReturn(rc, rc);
2476
2477 pVCpu->hm.s.vmx.u32ProcCtls = val;
2478
2479 /*
2480 * Secondary processor-based VM-execution controls.
2481 */
2482 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2483 {
2484 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2485 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2486
2487 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2488 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2489
2490 if (pVM->hm.s.fNestedPaging)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2492 else
2493 {
2494 /*
2495 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2496 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2497 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2498 */
2499 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2500 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2501 }
2502
2503 if (pVM->hm.s.vmx.fVpid)
2504 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2505
2506 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2507 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2508
2509 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2510 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2511 * done dynamically. */
2512 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2513 {
2514 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2515 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2516 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2517 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2518 AssertRCReturn(rc, rc);
2519 }
2520
2521 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2522 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2523
2524 if ((val & zap) != val)
2525 {
2526 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2527 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2528 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2529 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2530 }
2531
2532 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2533 AssertRCReturn(rc, rc);
2534
2535 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2536 }
2537 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2538 {
2539 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2540 "available\n"));
2541 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2542 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2543 }
2544
2545 return VINF_SUCCESS;
2546}
2547
2548
2549/**
2550 * Sets up miscellaneous (everything other than Pin & Processor-based
2551 * VM-execution) control fields in the VMCS.
2552 *
2553 * @returns VBox status code.
2554 * @param pVM Pointer to the VM.
2555 * @param pVCpu Pointer to the VMCPU.
2556 */
2557static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2558{
2559 NOREF(pVM);
2560 AssertPtr(pVM);
2561 AssertPtr(pVCpu);
2562
2563 int rc = VERR_GENERAL_FAILURE;
2564
2565 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2566#if 0
2567 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2568 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2569 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2570
2571 /*
2572 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2573 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2574 * We thus use the exception bitmap to control it rather than use both.
2575 */
2576 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2577 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2578
2579 /** @todo Explore possibility of using IO-bitmaps. */
2580 /* All IO & IOIO instructions cause VM-exits. */
2581 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2582 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2583
2584 /* Initialize the MSR-bitmap area. */
2585 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2586 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2587 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2588#endif
2589
2590 /* Setup MSR auto-load/store area. */
2591 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2592 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2593 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2594 AssertRCReturn(rc, rc);
2595 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2596 AssertRCReturn(rc, rc);
2597
2598 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2599 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2600 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2601 AssertRCReturn(rc, rc);
2602
2603 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2604 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2605 AssertRCReturn(rc, rc);
2606
2607 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2608#if 0
2609 /* Setup debug controls */
2610 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2611 AssertRCReturn(rc, rc);
2612 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2613 AssertRCReturn(rc, rc);
2614#endif
2615
2616 return rc;
2617}
2618
2619
2620/**
2621 * Sets up the initial exception bitmap in the VMCS based on static conditions
2622 * (i.e. conditions that cannot ever change after starting the VM).
2623 *
2624 * @returns VBox status code.
2625 * @param pVM Pointer to the VM.
2626 * @param pVCpu Pointer to the VMCPU.
2627 */
2628static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2629{
2630 AssertPtr(pVM);
2631 AssertPtr(pVCpu);
2632
2633 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2634
2635 uint32_t u32XcptBitmap = 0;
2636
2637 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2638 if (!pVM->hm.s.fNestedPaging)
2639 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2640
2641 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2642 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2643 AssertRCReturn(rc, rc);
2644 return rc;
2645}
2646
2647
2648/**
2649 * Sets up the initial guest-state mask. The guest-state mask is consulted
2650 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2651 * for the nested virtualization case (as it would cause a VM-exit).
2652 *
2653 * @param pVCpu Pointer to the VMCPU.
2654 */
2655static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2656{
2657 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2658 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2659 return VINF_SUCCESS;
2660}
2661
2662
2663/**
2664 * Does per-VM VT-x initialization.
2665 *
2666 * @returns VBox status code.
2667 * @param pVM Pointer to the VM.
2668 */
2669VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2670{
2671 LogFlowFunc(("pVM=%p\n", pVM));
2672
2673 int rc = hmR0VmxStructsAlloc(pVM);
2674 if (RT_FAILURE(rc))
2675 {
2676 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2677 return rc;
2678 }
2679
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x termination.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM Pointer to the VM.
2689 */
2690VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2695 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2696 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2697#endif
2698 hmR0VmxStructsFree(pVM);
2699 return VINF_SUCCESS;
2700}
2701
2702
2703/**
2704 * Sets up the VM for execution under VT-x.
2705 * This function is only called once per-VM during initialization.
2706 *
2707 * @returns VBox status code.
2708 * @param pVM Pointer to the VM.
2709 */
2710VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2711{
2712 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2713 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2714
2715 LogFlowFunc(("pVM=%p\n", pVM));
2716
2717 /*
2718 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2719 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2720 */
2721 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2722 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2723 || !pVM->hm.s.vmx.pRealModeTSS))
2724 {
2725 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2726 return VERR_INTERNAL_ERROR;
2727 }
2728
2729#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2730 /*
2731 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2732 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2733 */
2734 if ( pVM->hm.s.fAllow64BitGuests
2735 && !HMVMX_IS_64BIT_HOST_MODE())
2736 {
2737 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2738 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2739 }
2740#endif
2741
2742 /* Initialize these always, see hmR3InitFinalizeR0().*/
2743 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2744 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2745
2746 /* Setup the tagged-TLB flush handlers. */
2747 int rc = hmR0VmxSetupTaggedTlb(pVM);
2748 if (RT_FAILURE(rc))
2749 {
2750 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2751 return rc;
2752 }
2753
2754 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2755 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2756#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2757 if ( HMVMX_IS_64BIT_HOST_MODE()
2758 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2759 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2760 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2761 {
2762 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2763 }
2764#endif
2765
2766 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2767 {
2768 PVMCPU pVCpu = &pVM->aCpus[i];
2769 AssertPtr(pVCpu);
2770 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2771
2772 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2773 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2774
2775 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2776 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2777 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2778
2779 /* Set revision dword at the beginning of the VMCS structure. */
2780 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2781
2782 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2783 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2784 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2785 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2786
2787 /* Load this VMCS as the current VMCS. */
2788 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2789 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2790 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2791
2792 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2793 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2794 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2795
2796 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2797 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2798 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2799
2800 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2813 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2814 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2815 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2816#endif
2817
2818 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2819 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2820 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2821 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2822
2823 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2824
2825 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2826 }
2827
2828 return VINF_SUCCESS;
2829}
2830
2831
2832/**
2833 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2834 * the VMCS.
2835 *
2836 * @returns VBox status code.
2837 * @param pVM Pointer to the VM.
2838 * @param pVCpu Pointer to the VMCPU.
2839 */
2840DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2841{
2842 NOREF(pVM); NOREF(pVCpu);
2843
2844 RTCCUINTREG uReg = ASMGetCR0();
2845 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2846 AssertRCReturn(rc, rc);
2847
2848#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2849 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2850 if (HMVMX_IS_64BIT_HOST_MODE())
2851 {
2852 uint64_t uRegCR3 = HMR0Get64bitCR3();
2853 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2854 }
2855 else
2856#endif
2857 {
2858 uReg = ASMGetCR3();
2859 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2860 }
2861 AssertRCReturn(rc, rc);
2862
2863 uReg = ASMGetCR4();
2864 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2865 AssertRCReturn(rc, rc);
2866 return rc;
2867}
2868
2869
2870#if HC_ARCH_BITS == 64
2871/**
2872 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2873 * requirements. See hmR0VmxSaveHostSegmentRegs().
2874 */
2875# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2876 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2877 { \
2878 bool fValidSelector = true; \
2879 if ((selValue) & X86_SEL_LDT) \
2880 { \
2881 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2882 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2883 } \
2884 if (fValidSelector) \
2885 { \
2886 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2887 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2888 } \
2889 (selValue) = 0; \
2890 }
2891#endif
2892
2893
2894/**
2895 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2896 * the host-state area in the VMCS.
2897 *
2898 * @returns VBox status code.
2899 * @param pVM Pointer to the VM.
2900 * @param pVCpu Pointer to the VMCPU.
2901 */
2902DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2903{
2904 int rc = VERR_INTERNAL_ERROR_5;
2905
2906#if HC_ARCH_BITS == 64
2907 /*
2908 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2909 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2910 */
2911 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2912 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2913#endif
2914
2915 /*
2916 * Host DS, ES, FS and GS segment registers.
2917 */
2918#if HC_ARCH_BITS == 64
2919 RTSEL uSelDS = ASMGetDS();
2920 RTSEL uSelES = ASMGetES();
2921 RTSEL uSelFS = ASMGetFS();
2922 RTSEL uSelGS = ASMGetGS();
2923#else
2924 RTSEL uSelDS = 0;
2925 RTSEL uSelES = 0;
2926 RTSEL uSelFS = 0;
2927 RTSEL uSelGS = 0;
2928#endif
2929
2930 /* Recalculate which host-state bits need to be manually restored. */
2931 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2932
2933 /*
2934 * Host CS and SS segment registers.
2935 */
2936#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2937 RTSEL uSelCS;
2938 RTSEL uSelSS;
2939 if (HMVMX_IS_64BIT_HOST_MODE())
2940 {
2941 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2942 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2943 }
2944 else
2945 {
2946 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2947 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2948 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2949 }
2950#else
2951 RTSEL uSelCS = ASMGetCS();
2952 RTSEL uSelSS = ASMGetSS();
2953#endif
2954
2955 /*
2956 * Host TR segment register.
2957 */
2958 RTSEL uSelTR = ASMGetTR();
2959
2960#if HC_ARCH_BITS == 64
2961 /*
2962 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2963 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2964 */
2965 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2966 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2967 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2968 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2969# undef VMXLOCAL_ADJUST_HOST_SEG
2970#endif
2971
2972 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2973 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2974 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2975 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2976 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2977 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2978 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2979 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2980 Assert(uSelCS);
2981 Assert(uSelTR);
2982
2983 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2984#if 0
2985 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2986 Assert(uSelSS != 0);
2987#endif
2988
2989 /* Write these host selector fields into the host-state area in the VMCS. */
2990 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2991 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2992#if HC_ARCH_BITS == 64
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2994 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2995 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2997#endif
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2999
3000 /*
3001 * Host GDTR and IDTR.
3002 */
3003 RTGDTR Gdtr;
3004 RT_ZERO(Gdtr);
3005#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3006 if (HMVMX_IS_64BIT_HOST_MODE())
3007 {
3008 X86XDTR64 Gdtr64;
3009 X86XDTR64 Idtr64;
3010 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3011 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3012 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3013
3014 Gdtr.cbGdt = Gdtr64.cb;
3015 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3016 }
3017 else
3018#endif
3019 {
3020 RTIDTR Idtr;
3021 ASMGetGDTR(&Gdtr);
3022 ASMGetIDTR(&Idtr);
3023 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3024 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3025
3026#if HC_ARCH_BITS == 64
3027 /*
3028 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3029 * maximum limit (0xffff) on every VM-exit.
3030 */
3031 if (Gdtr.cbGdt != 0xffff)
3032 {
3033 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3034 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3035 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3036 }
3037
3038 /*
3039 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3040 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3041 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3042 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3043 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3044 * hosts where we are pretty sure it won't cause trouble.
3045 */
3046# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3047 if (Idtr.cbIdt < 0x0fff)
3048# else
3049 if (Idtr.cbIdt != 0xffff)
3050# endif
3051 {
3052 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3053 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3054 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3055 }
3056#endif
3057 }
3058
3059 /*
3060 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3061 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3062 */
3063 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3064 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3065 VERR_VMX_INVALID_HOST_STATE);
3066
3067 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3068#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3069 if (HMVMX_IS_64BIT_HOST_MODE())
3070 {
3071 /* We need the 64-bit TR base for hybrid darwin. */
3072 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3073 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3074 }
3075 else
3076#endif
3077 {
3078 uintptr_t uTRBase;
3079#if HC_ARCH_BITS == 64
3080 uTRBase = X86DESC64_BASE(pDesc);
3081
3082 /*
3083 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3084 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3085 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3086 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3087 *
3088 * [1] See Intel spec. 3.5 "System Descriptor Types".
3089 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3090 */
3091 Assert(pDesc->System.u4Type == 11);
3092 if ( pDesc->System.u16LimitLow != 0x67
3093 || pDesc->System.u4LimitHigh)
3094 {
3095 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3096 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3097 if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3098 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3099 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3100
3101 /* Store the GDTR here as we need it while restoring TR. */
3102 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3103 }
3104#else
3105 uTRBase = X86DESC_BASE(pDesc);
3106#endif
3107 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3108 }
3109 AssertRCReturn(rc, rc);
3110
3111 /*
3112 * Host FS base and GS base.
3113 */
3114#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3115 if (HMVMX_IS_64BIT_HOST_MODE())
3116 {
3117 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3118 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3119 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3120 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3121
3122# if HC_ARCH_BITS == 64
3123 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3124 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3125 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3126 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3127 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3128# endif
3129 }
3130#endif
3131 return rc;
3132}
3133
3134
3135/**
3136 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3137 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3138 * the host after every successful VM-exit.
3139 *
3140 * @returns VBox status code.
3141 * @param pVM Pointer to the VM.
3142 * @param pVCpu Pointer to the VMCPU.
3143 *
3144 * @remarks No-long-jump zone!!!
3145 */
3146DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3147{
3148 NOREF(pVM);
3149
3150 AssertPtr(pVCpu);
3151 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3152
3153 int rc = VINF_SUCCESS;
3154#if HC_ARCH_BITS == 64
3155 if (pVM->hm.s.fAllow64BitGuests)
3156 hmR0VmxLazySaveHostMsrs(pVCpu);
3157#endif
3158
3159 /*
3160 * Host Sysenter MSRs.
3161 */
3162 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3163 AssertRCReturn(rc, rc);
3164#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3165 if (HMVMX_IS_64BIT_HOST_MODE())
3166 {
3167 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3168 AssertRCReturn(rc, rc);
3169 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3170 }
3171 else
3172 {
3173 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3174 AssertRCReturn(rc, rc);
3175 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3176 }
3177#elif HC_ARCH_BITS == 32
3178 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3179 AssertRCReturn(rc, rc);
3180 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3181#else
3182 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3183 AssertRCReturn(rc, rc);
3184 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3185#endif
3186 AssertRCReturn(rc, rc);
3187
3188 /*
3189 * Host EFER MSR.
3190 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3191 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3192 */
3193 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3194 {
3195 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3196 AssertRCReturn(rc, rc);
3197 }
3198
3199 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3200 * hmR0VmxLoadGuestExitCtls() !! */
3201
3202 return rc;
3203}
3204
3205
3206/**
3207 * Figures out if we need to swap the EFER MSR which is
3208 * particularly expensive.
3209 *
3210 * We check all relevant bits. For now, that's everything
3211 * besides LMA/LME, as these two bits are handled by VM-entry,
3212 * see hmR0VmxLoadGuestExitCtls() and
3213 * hmR0VMxLoadGuestEntryCtls().
3214 *
3215 * @returns true if we need to load guest EFER, false otherwise.
3216 * @param pVCpu Pointer to the VMCPU.
3217 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3218 * out-of-sync. Make sure to update the required fields
3219 * before using them.
3220 *
3221 * @remarks Requires EFER, CR4.
3222 * @remarks No-long-jump zone!!!
3223 */
3224static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3225{
3226#ifdef HMVMX_ALWAYS_SWAP_EFER
3227 return true;
3228#endif
3229
3230#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3231 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3232 if (CPUMIsGuestInLongMode(pVCpu))
3233 return false;
3234#endif
3235
3236 PVM pVM = pVCpu->CTX_SUFF(pVM);
3237 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3238 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3239
3240 /*
3241 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3242 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3243 */
3244 if ( CPUMIsGuestInLongMode(pVCpu)
3245 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3246 {
3247 return true;
3248 }
3249
3250 /*
3251 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3252 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3253 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3254 */
3255 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3256 && (pMixedCtx->cr0 & X86_CR0_PG)
3257 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3258 {
3259 /* Assert that host is PAE capable. */
3260 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3261 return true;
3262 }
3263
3264 /** @todo Check the latest Intel spec. for any other bits,
3265 * like SMEP/SMAP? */
3266 return false;
3267}
3268
3269
3270/**
3271 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3272 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3273 * controls".
3274 *
3275 * @returns VBox status code.
3276 * @param pVCpu Pointer to the VMCPU.
3277 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3278 * out-of-sync. Make sure to update the required fields
3279 * before using them.
3280 *
3281 * @remarks Requires EFER.
3282 * @remarks No-long-jump zone!!!
3283 */
3284DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3285{
3286 int rc = VINF_SUCCESS;
3287 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3288 {
3289 PVM pVM = pVCpu->CTX_SUFF(pVM);
3290 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3291 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3292
3293 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3294 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3295
3296 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3297 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3298 {
3299 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3300 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3301 }
3302 else
3303 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3304
3305 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3306 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3307 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3308 {
3309 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3310 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3311 }
3312
3313 /*
3314 * The following should -not- be set (since we're not in SMM mode):
3315 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3316 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3317 */
3318
3319 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3320 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3321
3322 if ((val & zap) != val)
3323 {
3324 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3325 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3326 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3327 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3328 }
3329
3330 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3331 AssertRCReturn(rc, rc);
3332
3333 pVCpu->hm.s.vmx.u32EntryCtls = val;
3334 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3335 }
3336 return rc;
3337}
3338
3339
3340/**
3341 * Sets up the VM-exit controls in the VMCS.
3342 *
3343 * @returns VBox status code.
3344 * @param pVM Pointer to the VM.
3345 * @param pVCpu Pointer to the VMCPU.
3346 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3347 * out-of-sync. Make sure to update the required fields
3348 * before using them.
3349 *
3350 * @remarks Requires EFER.
3351 */
3352DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3353{
3354 NOREF(pMixedCtx);
3355
3356 int rc = VINF_SUCCESS;
3357 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3358 {
3359 PVM pVM = pVCpu->CTX_SUFF(pVM);
3360 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3361 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3362
3363 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3364 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3365
3366 /*
3367 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3368 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3369 */
3370#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3371 if (HMVMX_IS_64BIT_HOST_MODE())
3372 {
3373 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3374 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3375 }
3376 else
3377 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3378#else
3379 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3380 {
3381 /* The switcher returns to long mode, EFER is managed by the switcher. */
3382 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3383 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3384 }
3385 else
3386 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3387#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3388
3389 /* If the newer VMCS fields for managing EFER exists, use it. */
3390 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3391 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3392 {
3393 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3394 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3395 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3396 }
3397
3398 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3399 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3400
3401 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3402 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3403 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3404
3405 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3406 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3407
3408 if ((val & zap) != val)
3409 {
3410 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3411 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3412 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3413 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3414 }
3415
3416 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3417 AssertRCReturn(rc, rc);
3418
3419 pVCpu->hm.s.vmx.u32ExitCtls = val;
3420 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3421 }
3422 return rc;
3423}
3424
3425
3426/**
3427 * Loads the guest APIC and related state.
3428 *
3429 * @returns VBox status code.
3430 * @param pVM Pointer to the VM.
3431 * @param pVCpu Pointer to the VMCPU.
3432 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3433 * out-of-sync. Make sure to update the required fields
3434 * before using them.
3435 */
3436DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3437{
3438 NOREF(pMixedCtx);
3439
3440 int rc = VINF_SUCCESS;
3441 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3442 {
3443 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3444 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3445 {
3446 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3447
3448 bool fPendingIntr = false;
3449 uint8_t u8Tpr = 0;
3450 uint8_t u8PendingIntr = 0;
3451 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3452 AssertRCReturn(rc, rc);
3453
3454 /*
3455 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3456 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3457 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3458 * the interrupt when we VM-exit for other reasons.
3459 */
3460 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3461 uint32_t u32TprThreshold = 0;
3462 if (fPendingIntr)
3463 {
3464 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3465 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3466 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3467 if (u8PendingPriority <= u8TprPriority)
3468 u32TprThreshold = u8PendingPriority;
3469 else
3470 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3471 }
3472 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3473
3474 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3475 AssertRCReturn(rc, rc);
3476 }
3477
3478 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3479 }
3480 return rc;
3481}
3482
3483
3484/**
3485 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3486 *
3487 * @returns Guest's interruptibility-state.
3488 * @param pVCpu Pointer to the VMCPU.
3489 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3490 * out-of-sync. Make sure to update the required fields
3491 * before using them.
3492 *
3493 * @remarks No-long-jump zone!!!
3494 */
3495DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3496{
3497 /*
3498 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3499 */
3500 uint32_t uIntrState = 0;
3501 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3502 {
3503 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3504 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3505 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3506 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3507 {
3508 if (pMixedCtx->eflags.Bits.u1IF)
3509 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3510 else
3511 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3512 }
3513 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3514 }
3515
3516 /*
3517 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3518 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3519 * setting this would block host-NMIs and IRET will not clear the blocking.
3520 *
3521 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3522 */
3523 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3524 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3525 {
3526 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3527 }
3528
3529 return uIntrState;
3530}
3531
3532
3533/**
3534 * Loads the guest's interruptibility-state into the guest-state area in the
3535 * VMCS.
3536 *
3537 * @returns VBox status code.
3538 * @param pVCpu Pointer to the VMCPU.
3539 * @param uIntrState The interruptibility-state to set.
3540 */
3541static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3542{
3543 NOREF(pVCpu);
3544 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3545 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3546 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3547 AssertRCReturn(rc, rc);
3548 return rc;
3549}
3550
3551
3552/**
3553 * Loads the guest's RIP into the guest-state area in the VMCS.
3554 *
3555 * @returns VBox status code.
3556 * @param pVCpu Pointer to the VMCPU.
3557 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3558 * out-of-sync. Make sure to update the required fields
3559 * before using them.
3560 *
3561 * @remarks No-long-jump zone!!!
3562 */
3563static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3564{
3565 int rc = VINF_SUCCESS;
3566 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3567 {
3568 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3569 AssertRCReturn(rc, rc);
3570
3571 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3572 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3573 HMCPU_CF_VALUE(pVCpu)));
3574 }
3575 return rc;
3576}
3577
3578
3579/**
3580 * Loads the guest's RSP into the guest-state area in the VMCS.
3581 *
3582 * @returns VBox status code.
3583 * @param pVCpu Pointer to the VMCPU.
3584 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3585 * out-of-sync. Make sure to update the required fields
3586 * before using them.
3587 *
3588 * @remarks No-long-jump zone!!!
3589 */
3590static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3591{
3592 int rc = VINF_SUCCESS;
3593 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3594 {
3595 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3596 AssertRCReturn(rc, rc);
3597
3598 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3599 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3600 }
3601 return rc;
3602}
3603
3604
3605/**
3606 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3607 *
3608 * @returns VBox status code.
3609 * @param pVCpu Pointer to the VMCPU.
3610 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3611 * out-of-sync. Make sure to update the required fields
3612 * before using them.
3613 *
3614 * @remarks No-long-jump zone!!!
3615 */
3616static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3617{
3618 int rc = VINF_SUCCESS;
3619 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3620 {
3621 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3622 Let us assert it as such and use 32-bit VMWRITE. */
3623 Assert(!(pMixedCtx->rflags.u64 >> 32));
3624 X86EFLAGS Eflags = pMixedCtx->eflags;
3625 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3626 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3627 * These will never be cleared/set, unless some other part of the VMM
3628 * code is buggy - in which case we're better of finding and fixing
3629 * those bugs than hiding them. */
3630 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3631 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3632 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3633 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3634
3635 /*
3636 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3637 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3638 */
3639 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3640 {
3641 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3642 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3643 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3644 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3645 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3646 }
3647
3648 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3649 AssertRCReturn(rc, rc);
3650
3651 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3652 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3653 }
3654 return rc;
3655}
3656
3657
3658/**
3659 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3660 *
3661 * @returns VBox status code.
3662 * @param pVCpu Pointer to the VMCPU.
3663 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3664 * out-of-sync. Make sure to update the required fields
3665 * before using them.
3666 *
3667 * @remarks No-long-jump zone!!!
3668 */
3669DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3670{
3671 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3672 AssertRCReturn(rc, rc);
3673 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3674 AssertRCReturn(rc, rc);
3675 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3676 AssertRCReturn(rc, rc);
3677 return rc;
3678}
3679
3680
3681/**
3682 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3683 * CR0 is partially shared with the host and we have to consider the FPU bits.
3684 *
3685 * @returns VBox status code.
3686 * @param pVM Pointer to the VM.
3687 * @param pVCpu Pointer to the VMCPU.
3688 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3689 * out-of-sync. Make sure to update the required fields
3690 * before using them.
3691 *
3692 * @remarks No-long-jump zone!!!
3693 */
3694static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3695{
3696 /*
3697 * Guest CR0.
3698 * Guest FPU.
3699 */
3700 int rc = VINF_SUCCESS;
3701 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3702 {
3703 Assert(!(pMixedCtx->cr0 >> 32));
3704 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3705 PVM pVM = pVCpu->CTX_SUFF(pVM);
3706
3707 /* The guest's view (read access) of its CR0 is unblemished. */
3708 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3709 AssertRCReturn(rc, rc);
3710 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3711
3712 /* Setup VT-x's view of the guest CR0. */
3713 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3714 if (pVM->hm.s.fNestedPaging)
3715 {
3716 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3717 {
3718 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3719 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3720 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3721 }
3722 else
3723 {
3724 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3725 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3726 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3727 }
3728
3729 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3730 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3731 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3732
3733 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3734 AssertRCReturn(rc, rc);
3735 }
3736 else
3737 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3738
3739 /*
3740 * Guest FPU bits.
3741 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3742 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3743 */
3744 u32GuestCR0 |= X86_CR0_NE;
3745 bool fInterceptNM = false;
3746 if (CPUMIsGuestFPUStateActive(pVCpu))
3747 {
3748 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3749 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3750 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3751 }
3752 else
3753 {
3754 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3755 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3756 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3757 }
3758
3759 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3760 bool fInterceptMF = false;
3761 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3762 fInterceptMF = true;
3763
3764 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3765 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3766 {
3767 Assert(PDMVmmDevHeapIsEnabled(pVM));
3768 Assert(pVM->hm.s.vmx.pRealModeTSS);
3769 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3770 fInterceptNM = true;
3771 fInterceptMF = true;
3772 }
3773 else
3774 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3775
3776 if (fInterceptNM)
3777 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3778 else
3779 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3780
3781 if (fInterceptMF)
3782 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3783 else
3784 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3785
3786 /* Additional intercepts for debugging, define these yourself explicitly. */
3787#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3788 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3789 | RT_BIT(X86_XCPT_BP)
3790 | RT_BIT(X86_XCPT_DB)
3791 | RT_BIT(X86_XCPT_DE)
3792 | RT_BIT(X86_XCPT_NM)
3793 | RT_BIT(X86_XCPT_TS)
3794 | RT_BIT(X86_XCPT_UD)
3795 | RT_BIT(X86_XCPT_NP)
3796 | RT_BIT(X86_XCPT_SS)
3797 | RT_BIT(X86_XCPT_GP)
3798 | RT_BIT(X86_XCPT_PF)
3799 | RT_BIT(X86_XCPT_MF)
3800 ;
3801#elif defined(HMVMX_ALWAYS_TRAP_PF)
3802 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3803#endif
3804
3805 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3806
3807 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3808 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3809 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3810 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3811 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3812 else
3813 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3814
3815 u32GuestCR0 |= uSetCR0;
3816 u32GuestCR0 &= uZapCR0;
3817 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3818
3819 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3820 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3821 AssertRCReturn(rc, rc);
3822 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3823 AssertRCReturn(rc, rc);
3824 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3825 uZapCR0));
3826
3827 /*
3828 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3829 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3830 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3831 */
3832 uint32_t u32CR0Mask = 0;
3833 u32CR0Mask = X86_CR0_PE
3834 | X86_CR0_NE
3835 | X86_CR0_WP
3836 | X86_CR0_PG
3837 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3838 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3839 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3840
3841 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3842 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3843 * and @bugref{6944}. */
3844#if 0
3845 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3846 u32CR0Mask &= ~X86_CR0_PE;
3847#endif
3848 if (pVM->hm.s.fNestedPaging)
3849 u32CR0Mask &= ~X86_CR0_WP;
3850
3851 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3852 if (fInterceptNM)
3853 {
3854 u32CR0Mask |= X86_CR0_TS
3855 | X86_CR0_MP;
3856 }
3857
3858 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3859 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3860 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3861 AssertRCReturn(rc, rc);
3862 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3863
3864 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3865 }
3866 return rc;
3867}
3868
3869
3870/**
3871 * Loads the guest control registers (CR3, CR4) into the guest-state area
3872 * in the VMCS.
3873 *
3874 * @returns VBox status code.
3875 * @param pVM Pointer to the VM.
3876 * @param pVCpu Pointer to the VMCPU.
3877 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3878 * out-of-sync. Make sure to update the required fields
3879 * before using them.
3880 *
3881 * @remarks No-long-jump zone!!!
3882 */
3883static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3884{
3885 int rc = VINF_SUCCESS;
3886 PVM pVM = pVCpu->CTX_SUFF(pVM);
3887
3888 /*
3889 * Guest CR2.
3890 * It's always loaded in the assembler code. Nothing to do here.
3891 */
3892
3893 /*
3894 * Guest CR3.
3895 */
3896 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3897 {
3898 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3899 if (pVM->hm.s.fNestedPaging)
3900 {
3901 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3902
3903 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3904 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3905 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3906 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3907
3908 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3909 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3910 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3911
3912 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3913 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3914 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3915 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3916
3917 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3918 AssertRCReturn(rc, rc);
3919 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3920
3921 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3922 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3923 {
3924 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3925 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3926 {
3927 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3928 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3929 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3930 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3931 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3932 }
3933
3934 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3935 have Unrestricted Execution to handle the guest when it's not using paging. */
3936 GCPhysGuestCR3 = pMixedCtx->cr3;
3937 }
3938 else
3939 {
3940 /*
3941 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3942 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3943 * EPT takes care of translating it to host-physical addresses.
3944 */
3945 RTGCPHYS GCPhys;
3946 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3947 Assert(PDMVmmDevHeapIsEnabled(pVM));
3948
3949 /* We obtain it here every time as the guest could have relocated this PCI region. */
3950 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3951 AssertRCReturn(rc, rc);
3952
3953 GCPhysGuestCR3 = GCPhys;
3954 }
3955
3956 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3957 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3958 }
3959 else
3960 {
3961 /* Non-nested paging case, just use the hypervisor's CR3. */
3962 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3963
3964 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3965 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3966 }
3967 AssertRCReturn(rc, rc);
3968
3969 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3970 }
3971
3972 /*
3973 * Guest CR4.
3974 */
3975 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3976 {
3977 Assert(!(pMixedCtx->cr4 >> 32));
3978 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3979
3980 /* The guest's view of its CR4 is unblemished. */
3981 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3982 AssertRCReturn(rc, rc);
3983 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3984
3985 /* Setup VT-x's view of the guest CR4. */
3986 /*
3987 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3988 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3989 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3990 */
3991 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3992 {
3993 Assert(pVM->hm.s.vmx.pRealModeTSS);
3994 Assert(PDMVmmDevHeapIsEnabled(pVM));
3995 u32GuestCR4 &= ~X86_CR4_VME;
3996 }
3997
3998 if (pVM->hm.s.fNestedPaging)
3999 {
4000 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4001 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4002 {
4003 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4004 u32GuestCR4 |= X86_CR4_PSE;
4005 /* Our identity mapping is a 32-bit page directory. */
4006 u32GuestCR4 &= ~X86_CR4_PAE;
4007 }
4008 /* else use guest CR4.*/
4009 }
4010 else
4011 {
4012 /*
4013 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4014 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4015 */
4016 switch (pVCpu->hm.s.enmShadowMode)
4017 {
4018 case PGMMODE_REAL: /* Real-mode. */
4019 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4020 case PGMMODE_32_BIT: /* 32-bit paging. */
4021 {
4022 u32GuestCR4 &= ~X86_CR4_PAE;
4023 break;
4024 }
4025
4026 case PGMMODE_PAE: /* PAE paging. */
4027 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4028 {
4029 u32GuestCR4 |= X86_CR4_PAE;
4030 break;
4031 }
4032
4033 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4034 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4035#ifdef VBOX_ENABLE_64_BITS_GUESTS
4036 break;
4037#endif
4038 default:
4039 AssertFailed();
4040 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4041 }
4042 }
4043
4044 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4045 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4046 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4047 u32GuestCR4 |= uSetCR4;
4048 u32GuestCR4 &= uZapCR4;
4049
4050 /* Write VT-x's view of the guest CR4 into the VMCS. */
4051 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4052 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4053 AssertRCReturn(rc, rc);
4054
4055 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4056 uint32_t u32CR4Mask = 0;
4057 u32CR4Mask = X86_CR4_VME
4058 | X86_CR4_PAE
4059 | X86_CR4_PGE
4060 | X86_CR4_PSE
4061 | X86_CR4_VMXE;
4062 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4063 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4064 AssertRCReturn(rc, rc);
4065
4066 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4067 }
4068 return rc;
4069}
4070
4071
4072/**
4073 * Loads the guest debug registers into the guest-state area in the VMCS.
4074 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4075 *
4076 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4077 *
4078 * @returns VBox status code.
4079 * @param pVCpu Pointer to the VMCPU.
4080 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4081 * out-of-sync. Make sure to update the required fields
4082 * before using them.
4083 *
4084 * @remarks No-long-jump zone!!!
4085 */
4086static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4087{
4088 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4089 return VINF_SUCCESS;
4090
4091#ifdef VBOX_STRICT
4092 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4093 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4094 {
4095 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4096 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4097 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4098 }
4099#endif
4100
4101 int rc;
4102 PVM pVM = pVCpu->CTX_SUFF(pVM);
4103 bool fInterceptDB = false;
4104 bool fInterceptMovDRx = false;
4105 if ( pVCpu->hm.s.fSingleInstruction
4106 || DBGFIsStepping(pVCpu))
4107 {
4108 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4109 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4110 {
4111 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4112 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4113 AssertRCReturn(rc, rc);
4114 Assert(fInterceptDB == false);
4115 }
4116 else
4117 {
4118 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4119 pVCpu->hm.s.fClearTrapFlag = true;
4120 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4121 fInterceptDB = true;
4122 }
4123 }
4124
4125 if ( fInterceptDB
4126 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4127 {
4128 /*
4129 * Use the combined guest and host DRx values found in the hypervisor
4130 * register set because the debugger has breakpoints active or someone
4131 * is single stepping on the host side without a monitor trap flag.
4132 *
4133 * Note! DBGF expects a clean DR6 state before executing guest code.
4134 */
4135#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4136 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4137 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4138 {
4139 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4140 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4141 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4142 }
4143 else
4144#endif
4145 if (!CPUMIsHyperDebugStateActive(pVCpu))
4146 {
4147 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4148 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4149 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4150 }
4151
4152 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4153 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4154 AssertRCReturn(rc, rc);
4155
4156 pVCpu->hm.s.fUsingHyperDR7 = true;
4157 fInterceptDB = true;
4158 fInterceptMovDRx = true;
4159 }
4160 else
4161 {
4162 /*
4163 * If the guest has enabled debug registers, we need to load them prior to
4164 * executing guest code so they'll trigger at the right time.
4165 */
4166 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4167 {
4168#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4169 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4170 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4171 {
4172 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4173 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4174 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4175 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4176 }
4177 else
4178#endif
4179 if (!CPUMIsGuestDebugStateActive(pVCpu))
4180 {
4181 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4182 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4183 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4184 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4185 }
4186 Assert(!fInterceptDB);
4187 Assert(!fInterceptMovDRx);
4188 }
4189 /*
4190 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4191 * must intercept #DB in order to maintain a correct DR6 guest value.
4192 */
4193#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4194 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4195 && !CPUMIsGuestDebugStateActive(pVCpu))
4196#else
4197 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4198#endif
4199 {
4200 fInterceptMovDRx = true;
4201 fInterceptDB = true;
4202 }
4203
4204 /* Update guest DR7. */
4205 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4206 AssertRCReturn(rc, rc);
4207
4208 pVCpu->hm.s.fUsingHyperDR7 = false;
4209 }
4210
4211 /*
4212 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4213 */
4214 if (fInterceptDB)
4215 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4216 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4217 {
4218#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4219 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4220#endif
4221 }
4222 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4223 AssertRCReturn(rc, rc);
4224
4225 /*
4226 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4227 */
4228 if (fInterceptMovDRx)
4229 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4230 else
4231 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4232 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4233 AssertRCReturn(rc, rc);
4234
4235 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4236 return VINF_SUCCESS;
4237}
4238
4239
4240#ifdef VBOX_STRICT
4241/**
4242 * Strict function to validate segment registers.
4243 *
4244 * @remarks ASSUMES CR0 is up to date.
4245 */
4246static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4247{
4248 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4249 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4250 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4251 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4252 && ( !CPUMIsGuestInRealModeEx(pCtx)
4253 && !CPUMIsGuestInV86ModeEx(pCtx)))
4254 {
4255 /* Protected mode checks */
4256 /* CS */
4257 Assert(pCtx->cs.Attr.n.u1Present);
4258 Assert(!(pCtx->cs.Attr.u & 0xf00));
4259 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4260 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4261 || !(pCtx->cs.Attr.n.u1Granularity));
4262 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4263 || (pCtx->cs.Attr.n.u1Granularity));
4264 /* CS cannot be loaded with NULL in protected mode. */
4265 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4266 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4267 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4268 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4269 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4270 else
4271 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4272 /* SS */
4273 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4274 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4275 if ( !(pCtx->cr0 & X86_CR0_PE)
4276 || pCtx->cs.Attr.n.u4Type == 3)
4277 {
4278 Assert(!pCtx->ss.Attr.n.u2Dpl);
4279 }
4280 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4281 {
4282 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4283 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4284 Assert(pCtx->ss.Attr.n.u1Present);
4285 Assert(!(pCtx->ss.Attr.u & 0xf00));
4286 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4287 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4288 || !(pCtx->ss.Attr.n.u1Granularity));
4289 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4290 || (pCtx->ss.Attr.n.u1Granularity));
4291 }
4292 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4293 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4294 {
4295 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4296 Assert(pCtx->ds.Attr.n.u1Present);
4297 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4298 Assert(!(pCtx->ds.Attr.u & 0xf00));
4299 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4300 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4301 || !(pCtx->ds.Attr.n.u1Granularity));
4302 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4303 || (pCtx->ds.Attr.n.u1Granularity));
4304 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4305 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4306 }
4307 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4308 {
4309 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4310 Assert(pCtx->es.Attr.n.u1Present);
4311 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4312 Assert(!(pCtx->es.Attr.u & 0xf00));
4313 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4314 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4315 || !(pCtx->es.Attr.n.u1Granularity));
4316 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4317 || (pCtx->es.Attr.n.u1Granularity));
4318 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4319 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4320 }
4321 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4322 {
4323 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4324 Assert(pCtx->fs.Attr.n.u1Present);
4325 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4326 Assert(!(pCtx->fs.Attr.u & 0xf00));
4327 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4328 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4329 || !(pCtx->fs.Attr.n.u1Granularity));
4330 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4331 || (pCtx->fs.Attr.n.u1Granularity));
4332 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4333 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4334 }
4335 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4336 {
4337 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4338 Assert(pCtx->gs.Attr.n.u1Present);
4339 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4340 Assert(!(pCtx->gs.Attr.u & 0xf00));
4341 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4342 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4343 || !(pCtx->gs.Attr.n.u1Granularity));
4344 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4345 || (pCtx->gs.Attr.n.u1Granularity));
4346 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4347 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4348 }
4349 /* 64-bit capable CPUs. */
4350# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4351 if (HMVMX_IS_64BIT_HOST_MODE())
4352 {
4353 Assert(!(pCtx->cs.u64Base >> 32));
4354 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4355 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4356 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4357 }
4358# endif
4359 }
4360 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4361 || ( CPUMIsGuestInRealModeEx(pCtx)
4362 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4363 {
4364 /* Real and v86 mode checks. */
4365 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4366 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4367 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4368 {
4369 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4370 }
4371 else
4372 {
4373 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4374 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4375 }
4376
4377 /* CS */
4378 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4379 Assert(pCtx->cs.u32Limit == 0xffff);
4380 Assert(u32CSAttr == 0xf3);
4381 /* SS */
4382 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4383 Assert(pCtx->ss.u32Limit == 0xffff);
4384 Assert(u32SSAttr == 0xf3);
4385 /* DS */
4386 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4387 Assert(pCtx->ds.u32Limit == 0xffff);
4388 Assert(u32DSAttr == 0xf3);
4389 /* ES */
4390 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4391 Assert(pCtx->es.u32Limit == 0xffff);
4392 Assert(u32ESAttr == 0xf3);
4393 /* FS */
4394 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4395 Assert(pCtx->fs.u32Limit == 0xffff);
4396 Assert(u32FSAttr == 0xf3);
4397 /* GS */
4398 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4399 Assert(pCtx->gs.u32Limit == 0xffff);
4400 Assert(u32GSAttr == 0xf3);
4401 /* 64-bit capable CPUs. */
4402# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4403 if (HMVMX_IS_64BIT_HOST_MODE())
4404 {
4405 Assert(!(pCtx->cs.u64Base >> 32));
4406 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4407 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4408 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4409 }
4410# endif
4411 }
4412}
4413#endif /* VBOX_STRICT */
4414
4415
4416/**
4417 * Writes a guest segment register into the guest-state area in the VMCS.
4418 *
4419 * @returns VBox status code.
4420 * @param pVCpu Pointer to the VMCPU.
4421 * @param idxSel Index of the selector in the VMCS.
4422 * @param idxLimit Index of the segment limit in the VMCS.
4423 * @param idxBase Index of the segment base in the VMCS.
4424 * @param idxAccess Index of the access rights of the segment in the VMCS.
4425 * @param pSelReg Pointer to the segment selector.
4426 *
4427 * @remarks No-long-jump zone!!!
4428 */
4429static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4430 uint32_t idxAccess, PCPUMSELREG pSelReg)
4431{
4432 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4433 AssertRCReturn(rc, rc);
4434 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4435 AssertRCReturn(rc, rc);
4436 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4437 AssertRCReturn(rc, rc);
4438
4439 uint32_t u32Access = pSelReg->Attr.u;
4440 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4441 {
4442 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4443 u32Access = 0xf3;
4444 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4445 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4446 }
4447 else
4448 {
4449 /*
4450 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4451 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4452 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4453 * loaded in protected-mode have their attribute as 0.
4454 */
4455 if (!u32Access)
4456 u32Access = X86DESCATTR_UNUSABLE;
4457 }
4458
4459 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4460 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4461 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4462
4463 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4464 AssertRCReturn(rc, rc);
4465 return rc;
4466}
4467
4468
4469/**
4470 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4471 * into the guest-state area in the VMCS.
4472 *
4473 * @returns VBox status code.
4474 * @param pVM Pointer to the VM.
4475 * @param pVCPU Pointer to the VMCPU.
4476 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4477 * out-of-sync. Make sure to update the required fields
4478 * before using them.
4479 *
4480 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4481 * @remarks No-long-jump zone!!!
4482 */
4483static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4484{
4485 int rc = VERR_INTERNAL_ERROR_5;
4486 PVM pVM = pVCpu->CTX_SUFF(pVM);
4487
4488 /*
4489 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4490 */
4491 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4492 {
4493 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4494 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4495 {
4496 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4497 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4498 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4501 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4502 }
4503
4504#ifdef VBOX_WITH_REM
4505 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4506 {
4507 Assert(pVM->hm.s.vmx.pRealModeTSS);
4508 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4509 if ( pVCpu->hm.s.vmx.fWasInRealMode
4510 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4511 {
4512 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4513 in real-mode (e.g. OpenBSD 4.0) */
4514 REMFlushTBs(pVM);
4515 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4516 pVCpu->hm.s.vmx.fWasInRealMode = false;
4517 }
4518 }
4519#endif
4520 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4521 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4522 AssertRCReturn(rc, rc);
4523 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4524 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4525 AssertRCReturn(rc, rc);
4526 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4527 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4528 AssertRCReturn(rc, rc);
4529 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4530 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4531 AssertRCReturn(rc, rc);
4532 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4533 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4534 AssertRCReturn(rc, rc);
4535 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4536 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4537 AssertRCReturn(rc, rc);
4538
4539#ifdef VBOX_STRICT
4540 /* Validate. */
4541 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4542#endif
4543
4544 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4545 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4546 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4547 }
4548
4549 /*
4550 * Guest TR.
4551 */
4552 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4553 {
4554 /*
4555 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4556 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4557 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4558 */
4559 uint16_t u16Sel = 0;
4560 uint32_t u32Limit = 0;
4561 uint64_t u64Base = 0;
4562 uint32_t u32AccessRights = 0;
4563
4564 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4565 {
4566 u16Sel = pMixedCtx->tr.Sel;
4567 u32Limit = pMixedCtx->tr.u32Limit;
4568 u64Base = pMixedCtx->tr.u64Base;
4569 u32AccessRights = pMixedCtx->tr.Attr.u;
4570 }
4571 else
4572 {
4573 Assert(pVM->hm.s.vmx.pRealModeTSS);
4574 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4575
4576 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4577 RTGCPHYS GCPhys;
4578 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4579 AssertRCReturn(rc, rc);
4580
4581 X86DESCATTR DescAttr;
4582 DescAttr.u = 0;
4583 DescAttr.n.u1Present = 1;
4584 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4585
4586 u16Sel = 0;
4587 u32Limit = HM_VTX_TSS_SIZE;
4588 u64Base = GCPhys; /* in real-mode phys = virt. */
4589 u32AccessRights = DescAttr.u;
4590 }
4591
4592 /* Validate. */
4593 Assert(!(u16Sel & RT_BIT(2)));
4594 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4595 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4596 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4597 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4598 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4599 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4600 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4601 Assert( (u32Limit & 0xfff) == 0xfff
4602 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4603 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4604 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4605
4606 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4607 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4608 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4609 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4610
4611 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4612 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4613 }
4614
4615 /*
4616 * Guest GDTR.
4617 */
4618 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4619 {
4620 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4621 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4622
4623 /* Validate. */
4624 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4625
4626 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4627 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4628 }
4629
4630 /*
4631 * Guest LDTR.
4632 */
4633 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4634 {
4635 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4636 uint32_t u32Access = 0;
4637 if (!pMixedCtx->ldtr.Attr.u)
4638 u32Access = X86DESCATTR_UNUSABLE;
4639 else
4640 u32Access = pMixedCtx->ldtr.Attr.u;
4641
4642 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4643 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4644 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4645 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4646
4647 /* Validate. */
4648 if (!(u32Access & X86DESCATTR_UNUSABLE))
4649 {
4650 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4651 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4652 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4655 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4656 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4657 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4658 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4659 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4660 }
4661
4662 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4663 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4664 }
4665
4666 /*
4667 * Guest IDTR.
4668 */
4669 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4670 {
4671 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4672 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4673
4674 /* Validate. */
4675 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4676
4677 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4678 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4679 }
4680
4681 return VINF_SUCCESS;
4682}
4683
4684
4685/**
4686 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4687 * areas. These MSRs will automatically be loaded to the host CPU on every
4688 * successful VM-entry and stored from the host CPU on every successful VM-exit.
4689 *
4690 * This also creates/updates MSR slots for the host MSRs. The actual host
4691 * MSR values are -not- updated here for performance reasons. See
4692 * hmR0VmxSaveHostMsrs().
4693 *
4694 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4695 *
4696 * @returns VBox status code.
4697 * @param pVCpu Pointer to the VMCPU.
4698 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4699 * out-of-sync. Make sure to update the required fields
4700 * before using them.
4701 *
4702 * @remarks No-long-jump zone!!!
4703 */
4704static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4705{
4706 AssertPtr(pVCpu);
4707 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4708
4709 /*
4710 * MSRs that we use the auto-load/store MSR area in the VMCS.
4711 */
4712 PVM pVM = pVCpu->CTX_SUFF(pVM);
4713 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4714 {
4715 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4716#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4717 if (pVM->hm.s.fAllow64BitGuests)
4718 {
4719 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4720 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4721 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4722 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4723# ifdef DEBUG
4724 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4725 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4726 {
4727 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4728 pMsr->u64Value));
4729 }
4730# endif
4731 }
4732#endif
4733 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4734 }
4735
4736 /*
4737 * Guest Sysenter MSRs.
4738 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4739 * VM-exits on WRMSRs for these MSRs.
4740 */
4741 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4742 {
4743 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4744 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4745 }
4746
4747 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4748 {
4749 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4750 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4751 }
4752
4753 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4754 {
4755 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4756 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4757 }
4758
4759 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4760 {
4761 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4762 {
4763 /*
4764 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4765 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4766 */
4767 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4768 {
4769 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4770 AssertRCReturn(rc,rc);
4771 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4772 }
4773 else
4774 {
4775 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4776 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4777 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4778 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4779 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4780 }
4781 }
4782 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4783 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4784 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4785 }
4786
4787 return VINF_SUCCESS;
4788}
4789
4790
4791/**
4792 * Loads the guest activity state into the guest-state area in the VMCS.
4793 *
4794 * @returns VBox status code.
4795 * @param pVCpu Pointer to the VMCPU.
4796 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4797 * out-of-sync. Make sure to update the required fields
4798 * before using them.
4799 *
4800 * @remarks No-long-jump zone!!!
4801 */
4802static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4803{
4804 NOREF(pCtx);
4805 /** @todo See if we can make use of other states, e.g.
4806 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4807 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4808 {
4809 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4810 AssertRCReturn(rc, rc);
4811
4812 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4813 }
4814 return VINF_SUCCESS;
4815}
4816
4817
4818/**
4819 * Sets up the appropriate function to run guest code.
4820 *
4821 * @returns VBox status code.
4822 * @param pVCpu Pointer to the VMCPU.
4823 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4824 * out-of-sync. Make sure to update the required fields
4825 * before using them.
4826 *
4827 * @remarks No-long-jump zone!!!
4828 */
4829static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4830{
4831 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4832 {
4833#ifndef VBOX_ENABLE_64_BITS_GUESTS
4834 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4835#endif
4836 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4837#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4838 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4839 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4840 {
4841 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4842 {
4843 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4844 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4845 | HM_CHANGED_VMX_EXIT_CTLS
4846 | HM_CHANGED_VMX_ENTRY_CTLS
4847 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4848 }
4849 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4850 }
4851#else
4852 /* 64-bit host or hybrid host. */
4853 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4854#endif
4855 }
4856 else
4857 {
4858 /* Guest is not in long mode, use the 32-bit handler. */
4859#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4860 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4861 {
4862 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4863 {
4864 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4865 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4866 | HM_CHANGED_VMX_EXIT_CTLS
4867 | HM_CHANGED_VMX_ENTRY_CTLS
4868 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4869 }
4870 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4871 }
4872#else
4873 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4874#endif
4875 }
4876 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4877 return VINF_SUCCESS;
4878}
4879
4880
4881/**
4882 * Wrapper for running the guest code in VT-x.
4883 *
4884 * @returns VBox strict status code.
4885 * @param pVM Pointer to the VM.
4886 * @param pVCpu Pointer to the VMCPU.
4887 * @param pCtx Pointer to the guest-CPU context.
4888 *
4889 * @remarks No-long-jump zone!!!
4890 */
4891DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4892{
4893 /*
4894 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4895 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4896 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4897 */
4898 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4899 /** @todo Add stats for resume vs launch. */
4900#ifdef VBOX_WITH_KERNEL_USING_XMM
4901 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4902#else
4903 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4904#endif
4905}
4906
4907
4908/**
4909 * Reports world-switch error and dumps some useful debug info.
4910 *
4911 * @param pVM Pointer to the VM.
4912 * @param pVCpu Pointer to the VMCPU.
4913 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4914 * @param pCtx Pointer to the guest-CPU context.
4915 * @param pVmxTransient Pointer to the VMX transient structure (only
4916 * exitReason updated).
4917 */
4918static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4919{
4920 Assert(pVM);
4921 Assert(pVCpu);
4922 Assert(pCtx);
4923 Assert(pVmxTransient);
4924 HMVMX_ASSERT_PREEMPT_SAFE();
4925
4926 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4927 switch (rcVMRun)
4928 {
4929 case VERR_VMX_INVALID_VMXON_PTR:
4930 AssertFailed();
4931 break;
4932 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4933 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4934 {
4935 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4936 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4937 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4938 AssertRC(rc);
4939
4940 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4941 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4942 Cannot do it here as we may have been long preempted. */
4943
4944#ifdef VBOX_STRICT
4945 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4946 pVmxTransient->uExitReason));
4947 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4948 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4949 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4950 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4951 else
4952 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4953 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4954 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4955
4956 /* VMX control bits. */
4957 uint32_t u32Val;
4958 uint64_t u64Val;
4959 HMVMXHCUINTREG uHCReg;
4960 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4961 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4962 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4963 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4964 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4965 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4970 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4971 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4974 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4975 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4976 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4977 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4978 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4979 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4980 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4981 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4982 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4983 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4984 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4985 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4986 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4987 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4992 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4993 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4994 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4995 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4996 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4997 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4998 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4999 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5000 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5001 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5002
5003 /* Guest bits. */
5004 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5005 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5006 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5007 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5008 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5009 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5010 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5011 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5012
5013 /* Host bits. */
5014 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5015 Log4(("Host CR0 %#RHr\n", uHCReg));
5016 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5017 Log4(("Host CR3 %#RHr\n", uHCReg));
5018 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5019 Log4(("Host CR4 %#RHr\n", uHCReg));
5020
5021 RTGDTR HostGdtr;
5022 PCX86DESCHC pDesc;
5023 ASMGetGDTR(&HostGdtr);
5024 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5025 Log4(("Host CS %#08x\n", u32Val));
5026 if (u32Val < HostGdtr.cbGdt)
5027 {
5028 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5029 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5030 }
5031
5032 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5033 Log4(("Host DS %#08x\n", u32Val));
5034 if (u32Val < HostGdtr.cbGdt)
5035 {
5036 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5037 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5038 }
5039
5040 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5041 Log4(("Host ES %#08x\n", u32Val));
5042 if (u32Val < HostGdtr.cbGdt)
5043 {
5044 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5045 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5046 }
5047
5048 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5049 Log4(("Host FS %#08x\n", u32Val));
5050 if (u32Val < HostGdtr.cbGdt)
5051 {
5052 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5053 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5054 }
5055
5056 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5057 Log4(("Host GS %#08x\n", u32Val));
5058 if (u32Val < HostGdtr.cbGdt)
5059 {
5060 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5061 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5062 }
5063
5064 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5065 Log4(("Host SS %#08x\n", u32Val));
5066 if (u32Val < HostGdtr.cbGdt)
5067 {
5068 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5069 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5070 }
5071
5072 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5073 Log4(("Host TR %#08x\n", u32Val));
5074 if (u32Val < HostGdtr.cbGdt)
5075 {
5076 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5077 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5078 }
5079
5080 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5081 Log4(("Host TR Base %#RHv\n", uHCReg));
5082 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5083 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5084 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5085 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5086 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5087 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5088 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5089 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5091 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5092 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5093 Log4(("Host RSP %#RHv\n", uHCReg));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5095 Log4(("Host RIP %#RHv\n", uHCReg));
5096# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5097 if (HMVMX_IS_64BIT_HOST_MODE())
5098 {
5099 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5100 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5101 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5102 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5103 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5104 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5105 }
5106# endif
5107#endif /* VBOX_STRICT */
5108 break;
5109 }
5110
5111 default:
5112 /* Impossible */
5113 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5114 break;
5115 }
5116 NOREF(pVM); NOREF(pCtx);
5117}
5118
5119
5120#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5121#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5122# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5123#endif
5124#ifdef VBOX_STRICT
5125static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5126{
5127 switch (idxField)
5128 {
5129 case VMX_VMCS_GUEST_RIP:
5130 case VMX_VMCS_GUEST_RSP:
5131 case VMX_VMCS_GUEST_SYSENTER_EIP:
5132 case VMX_VMCS_GUEST_SYSENTER_ESP:
5133 case VMX_VMCS_GUEST_GDTR_BASE:
5134 case VMX_VMCS_GUEST_IDTR_BASE:
5135 case VMX_VMCS_GUEST_CS_BASE:
5136 case VMX_VMCS_GUEST_DS_BASE:
5137 case VMX_VMCS_GUEST_ES_BASE:
5138 case VMX_VMCS_GUEST_FS_BASE:
5139 case VMX_VMCS_GUEST_GS_BASE:
5140 case VMX_VMCS_GUEST_SS_BASE:
5141 case VMX_VMCS_GUEST_LDTR_BASE:
5142 case VMX_VMCS_GUEST_TR_BASE:
5143 case VMX_VMCS_GUEST_CR3:
5144 return true;
5145 }
5146 return false;
5147}
5148
5149static bool hmR0VmxIsValidReadField(uint32_t idxField)
5150{
5151 switch (idxField)
5152 {
5153 /* Read-only fields. */
5154 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5155 return true;
5156 }
5157 /* Remaining readable fields should also be writable. */
5158 return hmR0VmxIsValidWriteField(idxField);
5159}
5160#endif /* VBOX_STRICT */
5161
5162
5163/**
5164 * Executes the specified handler in 64-bit mode.
5165 *
5166 * @returns VBox status code.
5167 * @param pVM Pointer to the VM.
5168 * @param pVCpu Pointer to the VMCPU.
5169 * @param pCtx Pointer to the guest CPU context.
5170 * @param enmOp The operation to perform.
5171 * @param cbParam Number of parameters.
5172 * @param paParam Array of 32-bit parameters.
5173 */
5174VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5175 uint32_t *paParam)
5176{
5177 int rc, rc2;
5178 PHMGLOBALCPUINFO pCpu;
5179 RTHCPHYS HCPhysCpuPage;
5180 RTCCUINTREG uOldEflags;
5181
5182 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5183 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5184 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5185 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5186
5187#ifdef VBOX_STRICT
5188 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5189 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5190
5191 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5192 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5193#endif
5194
5195 /* Disable interrupts. */
5196 uOldEflags = ASMIntDisableFlags();
5197
5198#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5199 RTCPUID idHostCpu = RTMpCpuId();
5200 CPUMR0SetLApic(pVCpu, idHostCpu);
5201#endif
5202
5203 pCpu = HMR0GetCurrentCpu();
5204 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5205
5206 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5207 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5208
5209 /* Leave VMX Root Mode. */
5210 VMXDisable();
5211
5212 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5213
5214 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5215 CPUMSetHyperEIP(pVCpu, enmOp);
5216 for (int i = (int)cbParam - 1; i >= 0; i--)
5217 CPUMPushHyper(pVCpu, paParam[i]);
5218
5219 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5220
5221 /* Call the switcher. */
5222 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5223 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5224
5225 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5226 /* Make sure the VMX instructions don't cause #UD faults. */
5227 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5228
5229 /* Re-enter VMX Root Mode */
5230 rc2 = VMXEnable(HCPhysCpuPage);
5231 if (RT_FAILURE(rc2))
5232 {
5233 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5234 ASMSetFlags(uOldEflags);
5235 return rc2;
5236 }
5237
5238 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5239 AssertRC(rc2);
5240 Assert(!(ASMGetFlags() & X86_EFL_IF));
5241 ASMSetFlags(uOldEflags);
5242 return rc;
5243}
5244
5245
5246/**
5247 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5248 * supporting 64-bit guests.
5249 *
5250 * @returns VBox status code.
5251 * @param fResume Whether to VMLAUNCH or VMRESUME.
5252 * @param pCtx Pointer to the guest-CPU context.
5253 * @param pCache Pointer to the VMCS cache.
5254 * @param pVM Pointer to the VM.
5255 * @param pVCpu Pointer to the VMCPU.
5256 */
5257DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5258{
5259 uint32_t aParam[6];
5260 PHMGLOBALCPUINFO pCpu = NULL;
5261 RTHCPHYS HCPhysCpuPage = 0;
5262 int rc = VERR_INTERNAL_ERROR_5;
5263
5264 pCpu = HMR0GetCurrentCpu();
5265 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5266
5267#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5268 pCache->uPos = 1;
5269 pCache->interPD = PGMGetInterPaeCR3(pVM);
5270 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5271#endif
5272
5273#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5274 pCache->TestIn.HCPhysCpuPage = 0;
5275 pCache->TestIn.HCPhysVmcs = 0;
5276 pCache->TestIn.pCache = 0;
5277 pCache->TestOut.HCPhysVmcs = 0;
5278 pCache->TestOut.pCache = 0;
5279 pCache->TestOut.pCtx = 0;
5280 pCache->TestOut.eflags = 0;
5281#endif
5282
5283 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5284 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5285 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5286 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5287 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5288 aParam[5] = 0;
5289
5290#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5291 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5292 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5293#endif
5294 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5295
5296#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5297 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5298 Assert(pCtx->dr[4] == 10);
5299 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5300#endif
5301
5302#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5303 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5304 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5305 pVCpu->hm.s.vmx.HCPhysVmcs));
5306 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5307 pCache->TestOut.HCPhysVmcs));
5308 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5309 pCache->TestOut.pCache));
5310 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5311 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5312 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5313 pCache->TestOut.pCtx));
5314 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5315#endif
5316 return rc;
5317}
5318
5319
5320/**
5321 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5322 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5323 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5324 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5325 *
5326 * @returns VBox status code.
5327 * @param pVM Pointer to the VM.
5328 * @param pVCpu Pointer to the VMCPU.
5329 */
5330static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5331{
5332#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5333{ \
5334 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5335 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5336 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5337 ++cReadFields; \
5338}
5339
5340 AssertPtr(pVM);
5341 AssertPtr(pVCpu);
5342 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5343 uint32_t cReadFields = 0;
5344
5345 /*
5346 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5347 * and serve to indicate exceptions to the rules.
5348 */
5349
5350 /* Guest-natural selector base fields. */
5351#if 0
5352 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5353 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5354 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5355#endif
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5357 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5368#if 0
5369 /* Unused natural width guest-state fields. */
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5371 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5372#endif
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5375
5376 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5377#if 0
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5387#endif
5388
5389 /* Natural width guest-state fields. */
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5391#if 0
5392 /* Currently unused field. */
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5394#endif
5395
5396 if (pVM->hm.s.fNestedPaging)
5397 {
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5399 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5400 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5401 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5402 }
5403 else
5404 {
5405 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5406 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5407 }
5408
5409#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5410 return VINF_SUCCESS;
5411}
5412
5413
5414/**
5415 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5416 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5417 * darwin, running 64-bit guests).
5418 *
5419 * @returns VBox status code.
5420 * @param pVCpu Pointer to the VMCPU.
5421 * @param idxField The VMCS field encoding.
5422 * @param u64Val 16, 32 or 64-bit value.
5423 */
5424VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5425{
5426 int rc;
5427 switch (idxField)
5428 {
5429 /*
5430 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5431 */
5432 /* 64-bit Control fields. */
5433 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5434 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5435 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5436 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5437 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5438 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5439 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5440 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5441 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5442 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5443 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5444 case VMX_VMCS64_CTRL_EPTP_FULL:
5445 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5446 /* 64-bit Guest-state fields. */
5447 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5448 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5449 case VMX_VMCS64_GUEST_PAT_FULL:
5450 case VMX_VMCS64_GUEST_EFER_FULL:
5451 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5452 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5453 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5454 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5455 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5456 /* 64-bit Host-state fields. */
5457 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5458 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5459 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5460 {
5461 rc = VMXWriteVmcs32(idxField, u64Val);
5462 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5463 break;
5464 }
5465
5466 /*
5467 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5468 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5469 */
5470 /* Natural-width Guest-state fields. */
5471 case VMX_VMCS_GUEST_CR3:
5472 case VMX_VMCS_GUEST_ES_BASE:
5473 case VMX_VMCS_GUEST_CS_BASE:
5474 case VMX_VMCS_GUEST_SS_BASE:
5475 case VMX_VMCS_GUEST_DS_BASE:
5476 case VMX_VMCS_GUEST_FS_BASE:
5477 case VMX_VMCS_GUEST_GS_BASE:
5478 case VMX_VMCS_GUEST_LDTR_BASE:
5479 case VMX_VMCS_GUEST_TR_BASE:
5480 case VMX_VMCS_GUEST_GDTR_BASE:
5481 case VMX_VMCS_GUEST_IDTR_BASE:
5482 case VMX_VMCS_GUEST_RSP:
5483 case VMX_VMCS_GUEST_RIP:
5484 case VMX_VMCS_GUEST_SYSENTER_ESP:
5485 case VMX_VMCS_GUEST_SYSENTER_EIP:
5486 {
5487 if (!(u64Val >> 32))
5488 {
5489 /* If this field is 64-bit, VT-x will zero out the top bits. */
5490 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5491 }
5492 else
5493 {
5494 /* Assert that only the 32->64 switcher case should ever come here. */
5495 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5496 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5497 }
5498 break;
5499 }
5500
5501 default:
5502 {
5503 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5504 rc = VERR_INVALID_PARAMETER;
5505 break;
5506 }
5507 }
5508 AssertRCReturn(rc, rc);
5509 return rc;
5510}
5511
5512
5513/**
5514 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5515 * hosts (except darwin) for 64-bit guests.
5516 *
5517 * @param pVCpu Pointer to the VMCPU.
5518 * @param idxField The VMCS field encoding.
5519 * @param u64Val 16, 32 or 64-bit value.
5520 */
5521VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5522{
5523 AssertPtr(pVCpu);
5524 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5525
5526 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5527 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5528
5529 /* Make sure there are no duplicates. */
5530 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5531 {
5532 if (pCache->Write.aField[i] == idxField)
5533 {
5534 pCache->Write.aFieldVal[i] = u64Val;
5535 return VINF_SUCCESS;
5536 }
5537 }
5538
5539 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5540 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5541 pCache->Write.cValidEntries++;
5542 return VINF_SUCCESS;
5543}
5544
5545/* Enable later when the assembly code uses these as callbacks. */
5546#if 0
5547/*
5548 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5549 *
5550 * @param pVCpu Pointer to the VMCPU.
5551 * @param pCache Pointer to the VMCS cache.
5552 *
5553 * @remarks No-long-jump zone!!!
5554 */
5555VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5556{
5557 AssertPtr(pCache);
5558 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5559 {
5560 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5561 AssertRC(rc);
5562 }
5563 pCache->Write.cValidEntries = 0;
5564}
5565
5566
5567/**
5568 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5569 *
5570 * @param pVCpu Pointer to the VMCPU.
5571 * @param pCache Pointer to the VMCS cache.
5572 *
5573 * @remarks No-long-jump zone!!!
5574 */
5575VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5576{
5577 AssertPtr(pCache);
5578 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5579 {
5580 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5581 AssertRC(rc);
5582 }
5583}
5584#endif
5585#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5586
5587
5588/**
5589 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5590 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5591 * timer.
5592 *
5593 * @returns VBox status code.
5594 * @param pVCpu Pointer to the VMCPU.
5595 *
5596 * @remarks No-long-jump zone!!!
5597 */
5598static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5599{
5600 int rc = VERR_INTERNAL_ERROR_5;
5601 bool fOffsettedTsc = false;
5602 bool fParavirtTsc = false;
5603 PVM pVM = pVCpu->CTX_SUFF(pVM);
5604 if (pVM->hm.s.vmx.fUsePreemptTimer)
5605 {
5606 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &fParavirtTsc,
5607 &pVCpu->hm.s.vmx.u64TSCOffset);
5608
5609 /* Make sure the returned values have sane upper and lower boundaries. */
5610 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5611 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5612 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5613 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5614
5615 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5616 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5617 }
5618 else
5619 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5620
5621 if (fParavirtTsc)
5622 {
5623 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5624 AssertRC(rc);
5625 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5626 }
5627
5628 if (fOffsettedTsc)
5629 {
5630 uint64_t u64CurTSC = ASMReadTSC();
5631 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5632 {
5633 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5634 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5635
5636 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5637 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5638 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5639 }
5640 else
5641 {
5642 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5643 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5644 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5645 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5646 }
5647 }
5648 else
5649 {
5650 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5651 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5652 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5653 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5654 }
5655}
5656
5657
5658/**
5659 * Determines if an exception is a contributory exception. Contributory
5660 * exceptions are ones which can cause double-faults. Page-fault is
5661 * intentionally not included here as it's a conditional contributory exception.
5662 *
5663 * @returns true if the exception is contributory, false otherwise.
5664 * @param uVector The exception vector.
5665 */
5666DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5667{
5668 switch (uVector)
5669 {
5670 case X86_XCPT_GP:
5671 case X86_XCPT_SS:
5672 case X86_XCPT_NP:
5673 case X86_XCPT_TS:
5674 case X86_XCPT_DE:
5675 return true;
5676 default:
5677 break;
5678 }
5679 return false;
5680}
5681
5682
5683/**
5684 * Sets an event as a pending event to be injected into the guest.
5685 *
5686 * @param pVCpu Pointer to the VMCPU.
5687 * @param u32IntInfo The VM-entry interruption-information field.
5688 * @param cbInstr The VM-entry instruction length in bytes (for software
5689 * interrupts, exceptions and privileged software
5690 * exceptions).
5691 * @param u32ErrCode The VM-entry exception error code.
5692 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5693 * page-fault.
5694 *
5695 * @remarks Statistics counter assumes this is a guest event being injected or
5696 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5697 * always incremented.
5698 */
5699DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5700 RTGCUINTPTR GCPtrFaultAddress)
5701{
5702 Assert(!pVCpu->hm.s.Event.fPending);
5703 pVCpu->hm.s.Event.fPending = true;
5704 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5705 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5706 pVCpu->hm.s.Event.cbInstr = cbInstr;
5707 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5708
5709 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5710}
5711
5712
5713/**
5714 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5715 *
5716 * @param pVCpu Pointer to the VMCPU.
5717 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5718 * out-of-sync. Make sure to update the required fields
5719 * before using them.
5720 */
5721DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5722{
5723 NOREF(pMixedCtx);
5724 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5725 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5726 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5727 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5728}
5729
5730
5731/**
5732 * Handle a condition that occurred while delivering an event through the guest
5733 * IDT.
5734 *
5735 * @returns VBox status code (informational error codes included).
5736 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5737 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5738 * continue execution of the guest which will delivery the #DF.
5739 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5740 *
5741 * @param pVCpu Pointer to the VMCPU.
5742 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5743 * out-of-sync. Make sure to update the required fields
5744 * before using them.
5745 * @param pVmxTransient Pointer to the VMX transient structure.
5746 *
5747 * @remarks No-long-jump zone!!!
5748 */
5749static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5750{
5751 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5752
5753 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5754 AssertRCReturn(rc, rc);
5755 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5756 {
5757 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5758 AssertRCReturn(rc, rc);
5759
5760 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5761 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5762
5763 typedef enum
5764 {
5765 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5766 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5767 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5768 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5769 } VMXREFLECTXCPT;
5770
5771 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5772 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5773 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5774 {
5775 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5776 {
5777 enmReflect = VMXREFLECTXCPT_XCPT;
5778#ifdef VBOX_STRICT
5779 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5780 && uExitVector == X86_XCPT_PF)
5781 {
5782 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5783 }
5784#endif
5785 if ( uExitVector == X86_XCPT_PF
5786 && uIdtVector == X86_XCPT_PF)
5787 {
5788 pVmxTransient->fVectoringDoublePF = true;
5789 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5790 }
5791 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5792 && hmR0VmxIsContributoryXcpt(uExitVector)
5793 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5794 || uIdtVector == X86_XCPT_PF))
5795 {
5796 enmReflect = VMXREFLECTXCPT_DF;
5797 }
5798 else if (uIdtVector == X86_XCPT_DF)
5799 enmReflect = VMXREFLECTXCPT_TF;
5800 }
5801 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5802 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5803 {
5804 /*
5805 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5806 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5807 */
5808 enmReflect = VMXREFLECTXCPT_XCPT;
5809
5810 if (uExitVector == X86_XCPT_PF)
5811 {
5812 pVmxTransient->fVectoringPF = true;
5813 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5814 }
5815 }
5816 }
5817 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5818 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5819 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5820 {
5821 /*
5822 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5823 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5824 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5825 */
5826 enmReflect = VMXREFLECTXCPT_XCPT;
5827 }
5828
5829 /*
5830 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5831 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5832 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5833 *
5834 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5835 */
5836 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5837 && enmReflect == VMXREFLECTXCPT_XCPT
5838 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5839 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5840 {
5841 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5842 }
5843
5844 switch (enmReflect)
5845 {
5846 case VMXREFLECTXCPT_XCPT:
5847 {
5848 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5849 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5850 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5851
5852 uint32_t u32ErrCode = 0;
5853 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5854 {
5855 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5856 AssertRCReturn(rc, rc);
5857 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5858 }
5859
5860 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5861 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5862 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5863 rc = VINF_SUCCESS;
5864 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5865 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5866
5867 break;
5868 }
5869
5870 case VMXREFLECTXCPT_DF:
5871 {
5872 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5873 rc = VINF_HM_DOUBLE_FAULT;
5874 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5875 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5876
5877 break;
5878 }
5879
5880 case VMXREFLECTXCPT_TF:
5881 {
5882 rc = VINF_EM_RESET;
5883 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5884 uExitVector));
5885 break;
5886 }
5887
5888 default:
5889 Assert(rc == VINF_SUCCESS);
5890 break;
5891 }
5892 }
5893 else if ( VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5894 && uExitVector != X86_XCPT_DF
5895 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5896 {
5897 /*
5898 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5899 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5900 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5901 */
5902 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5903 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5904 }
5905
5906 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5907 return rc;
5908}
5909
5910
5911/**
5912 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5913 *
5914 * @returns VBox status code.
5915 * @param pVCpu Pointer to the VMCPU.
5916 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5917 * out-of-sync. Make sure to update the required fields
5918 * before using them.
5919 *
5920 * @remarks No-long-jump zone!!!
5921 */
5922static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5923{
5924 NOREF(pMixedCtx);
5925
5926 /*
5927 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5928 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5929 */
5930 VMMRZCallRing3Disable(pVCpu);
5931 HM_DISABLE_PREEMPT_IF_NEEDED();
5932
5933 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5934 {
5935 uint32_t uVal = 0;
5936 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5937 AssertRCReturn(rc, rc);
5938
5939 uint32_t uShadow = 0;
5940 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5941 AssertRCReturn(rc, rc);
5942
5943 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5944 CPUMSetGuestCR0(pVCpu, uVal);
5945 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5946 }
5947
5948 HM_RESTORE_PREEMPT_IF_NEEDED();
5949 VMMRZCallRing3Enable(pVCpu);
5950 return VINF_SUCCESS;
5951}
5952
5953
5954/**
5955 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5956 *
5957 * @returns VBox status code.
5958 * @param pVCpu Pointer to the VMCPU.
5959 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5960 * out-of-sync. Make sure to update the required fields
5961 * before using them.
5962 *
5963 * @remarks No-long-jump zone!!!
5964 */
5965static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5966{
5967 NOREF(pMixedCtx);
5968
5969 int rc = VINF_SUCCESS;
5970 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5971 {
5972 uint32_t uVal = 0;
5973 uint32_t uShadow = 0;
5974 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5975 AssertRCReturn(rc, rc);
5976 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5977 AssertRCReturn(rc, rc);
5978
5979 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5980 CPUMSetGuestCR4(pVCpu, uVal);
5981 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5982 }
5983 return rc;
5984}
5985
5986
5987/**
5988 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5989 *
5990 * @returns VBox status code.
5991 * @param pVCpu Pointer to the VMCPU.
5992 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5993 * out-of-sync. Make sure to update the required fields
5994 * before using them.
5995 *
5996 * @remarks No-long-jump zone!!!
5997 */
5998static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5999{
6000 int rc = VINF_SUCCESS;
6001 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6002 {
6003 uint64_t u64Val = 0;
6004 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6005 AssertRCReturn(rc, rc);
6006
6007 pMixedCtx->rip = u64Val;
6008 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6009 }
6010 return rc;
6011}
6012
6013
6014/**
6015 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6016 *
6017 * @returns VBox status code.
6018 * @param pVCpu Pointer to the VMCPU.
6019 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6020 * out-of-sync. Make sure to update the required fields
6021 * before using them.
6022 *
6023 * @remarks No-long-jump zone!!!
6024 */
6025static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6026{
6027 int rc = VINF_SUCCESS;
6028 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6029 {
6030 uint64_t u64Val = 0;
6031 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6032 AssertRCReturn(rc, rc);
6033
6034 pMixedCtx->rsp = u64Val;
6035 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6036 }
6037 return rc;
6038}
6039
6040
6041/**
6042 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6043 *
6044 * @returns VBox status code.
6045 * @param pVCpu Pointer to the VMCPU.
6046 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6047 * out-of-sync. Make sure to update the required fields
6048 * before using them.
6049 *
6050 * @remarks No-long-jump zone!!!
6051 */
6052static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6053{
6054 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6055 {
6056 uint32_t uVal = 0;
6057 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6058 AssertRCReturn(rc, rc);
6059
6060 pMixedCtx->eflags.u32 = uVal;
6061 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6062 {
6063 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6064 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6065
6066 pMixedCtx->eflags.Bits.u1VM = 0;
6067 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6068 }
6069
6070 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6071 }
6072 return VINF_SUCCESS;
6073}
6074
6075
6076/**
6077 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6078 * guest-CPU context.
6079 */
6080DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6081{
6082 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6083 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6084 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6085 return rc;
6086}
6087
6088
6089/**
6090 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6091 * from the guest-state area in the VMCS.
6092 *
6093 * @param pVCpu Pointer to the VMCPU.
6094 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6095 * out-of-sync. Make sure to update the required fields
6096 * before using them.
6097 *
6098 * @remarks No-long-jump zone!!!
6099 */
6100static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6101{
6102 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6103 {
6104 uint32_t uIntrState = 0;
6105 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6106 AssertRC(rc);
6107
6108 if (!uIntrState)
6109 {
6110 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6111 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6112
6113 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6114 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6115 }
6116 else
6117 {
6118 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6119 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6120 {
6121 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6122 AssertRC(rc);
6123 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6124 AssertRC(rc);
6125
6126 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6127 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6128 }
6129 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6130 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6131
6132 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6133 {
6134 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6135 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6136 }
6137 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6138 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6139 }
6140
6141 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6142 }
6143}
6144
6145
6146/**
6147 * Saves the guest's activity state.
6148 *
6149 * @returns VBox status code.
6150 * @param pVCpu Pointer to the VMCPU.
6151 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6152 * out-of-sync. Make sure to update the required fields
6153 * before using them.
6154 *
6155 * @remarks No-long-jump zone!!!
6156 */
6157static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6158{
6159 NOREF(pMixedCtx);
6160 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6161 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6162 return VINF_SUCCESS;
6163}
6164
6165
6166/**
6167 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6168 * the current VMCS into the guest-CPU context.
6169 *
6170 * @returns VBox status code.
6171 * @param pVCpu Pointer to the VMCPU.
6172 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6173 * out-of-sync. Make sure to update the required fields
6174 * before using them.
6175 *
6176 * @remarks No-long-jump zone!!!
6177 */
6178static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6179{
6180 int rc = VINF_SUCCESS;
6181 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6182 {
6183 uint32_t u32Val = 0;
6184 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6185 pMixedCtx->SysEnter.cs = u32Val;
6186 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6187 }
6188
6189 uint64_t u64Val = 0;
6190 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6191 {
6192 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6193 pMixedCtx->SysEnter.eip = u64Val;
6194 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6195 }
6196 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6197 {
6198 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6199 pMixedCtx->SysEnter.esp = u64Val;
6200 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6201 }
6202 return rc;
6203}
6204
6205
6206/**
6207 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6208 * the CPU back into the guest-CPU context.
6209 *
6210 * @returns VBox status code.
6211 * @param pVCpu Pointer to the VMCPU.
6212 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6213 * out-of-sync. Make sure to update the required fields
6214 * before using them.
6215 *
6216 * @remarks No-long-jump zone!!!
6217 */
6218static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6219{
6220#if HC_ARCH_BITS == 64
6221 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6222 {
6223 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6224 VMMRZCallRing3Disable(pVCpu);
6225 HM_DISABLE_PREEMPT_IF_NEEDED();
6226
6227 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6228 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6229 {
6230 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6231 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6232 }
6233
6234 HM_RESTORE_PREEMPT_IF_NEEDED();
6235 VMMRZCallRing3Enable(pVCpu);
6236 }
6237 else
6238 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6239#else
6240 NOREF(pMixedCtx);
6241 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6242#endif
6243
6244 return VINF_SUCCESS;
6245}
6246
6247
6248/**
6249 * Saves the auto load/store'd guest MSRs from the current VMCS into
6250 * the guest-CPU context.
6251 *
6252 * @returns VBox status code.
6253 * @param pVCpu Pointer to the VMCPU.
6254 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6255 * out-of-sync. Make sure to update the required fields
6256 * before using them.
6257 *
6258 * @remarks No-long-jump zone!!!
6259 */
6260static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6261{
6262 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6263 return VINF_SUCCESS;
6264
6265 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6266 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6267 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6268 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6269 {
6270 switch (pMsr->u32Msr)
6271 {
6272 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6273 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6274 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6275 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6276 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6277 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6278 break;
6279
6280 default:
6281 {
6282 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6283 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6284 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6285 }
6286 }
6287 }
6288
6289 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6290 return VINF_SUCCESS;
6291}
6292
6293
6294/**
6295 * Saves the guest control registers from the current VMCS into the guest-CPU
6296 * context.
6297 *
6298 * @returns VBox status code.
6299 * @param pVCpu Pointer to the VMCPU.
6300 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6301 * out-of-sync. Make sure to update the required fields
6302 * before using them.
6303 *
6304 * @remarks No-long-jump zone!!!
6305 */
6306static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6307{
6308 /* Guest CR0. Guest FPU. */
6309 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6310 AssertRCReturn(rc, rc);
6311
6312 /* Guest CR4. */
6313 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6314 AssertRCReturn(rc, rc);
6315
6316 /* Guest CR2 - updated always during the world-switch or in #PF. */
6317 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6318 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6319 {
6320 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6321 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6322
6323 PVM pVM = pVCpu->CTX_SUFF(pVM);
6324 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6325 || ( pVM->hm.s.fNestedPaging
6326 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6327 {
6328 uint64_t u64Val = 0;
6329 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6330 if (pMixedCtx->cr3 != u64Val)
6331 {
6332 CPUMSetGuestCR3(pVCpu, u64Val);
6333 if (VMMRZCallRing3IsEnabled(pVCpu))
6334 {
6335 PGMUpdateCR3(pVCpu, u64Val);
6336 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6337 }
6338 else
6339 {
6340 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6341 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6342 }
6343 }
6344
6345 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6346 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6347 {
6348 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6349 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6350 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6351 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6352
6353 if (VMMRZCallRing3IsEnabled(pVCpu))
6354 {
6355 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6356 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6357 }
6358 else
6359 {
6360 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6361 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6362 }
6363 }
6364 }
6365
6366 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6367 }
6368
6369 /*
6370 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6371 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6372 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6373 *
6374 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6375 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6376 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6377 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6378 *
6379 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6380 */
6381 if (VMMRZCallRing3IsEnabled(pVCpu))
6382 {
6383 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6384 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6385
6386 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6387 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6388
6389 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6390 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6391 }
6392
6393 return rc;
6394}
6395
6396
6397/**
6398 * Reads a guest segment register from the current VMCS into the guest-CPU
6399 * context.
6400 *
6401 * @returns VBox status code.
6402 * @param pVCpu Pointer to the VMCPU.
6403 * @param idxSel Index of the selector in the VMCS.
6404 * @param idxLimit Index of the segment limit in the VMCS.
6405 * @param idxBase Index of the segment base in the VMCS.
6406 * @param idxAccess Index of the access rights of the segment in the VMCS.
6407 * @param pSelReg Pointer to the segment selector.
6408 *
6409 * @remarks No-long-jump zone!!!
6410 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6411 * macro as that takes care of whether to read from the VMCS cache or
6412 * not.
6413 */
6414DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6415 PCPUMSELREG pSelReg)
6416{
6417 NOREF(pVCpu);
6418
6419 uint32_t u32Val = 0;
6420 int rc = VMXReadVmcs32(idxSel, &u32Val);
6421 AssertRCReturn(rc, rc);
6422 pSelReg->Sel = (uint16_t)u32Val;
6423 pSelReg->ValidSel = (uint16_t)u32Val;
6424 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6425
6426 rc = VMXReadVmcs32(idxLimit, &u32Val);
6427 AssertRCReturn(rc, rc);
6428 pSelReg->u32Limit = u32Val;
6429
6430 uint64_t u64Val = 0;
6431 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6432 AssertRCReturn(rc, rc);
6433 pSelReg->u64Base = u64Val;
6434
6435 rc = VMXReadVmcs32(idxAccess, &u32Val);
6436 AssertRCReturn(rc, rc);
6437 pSelReg->Attr.u = u32Val;
6438
6439 /*
6440 * If VT-x marks the segment as unusable, most other bits remain undefined:
6441 * - For CS the L, D and G bits have meaning.
6442 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6443 * - For the remaining data segments no bits are defined.
6444 *
6445 * The present bit and the unusable bit has been observed to be set at the
6446 * same time (the selector was supposed to be invalid as we started executing
6447 * a V8086 interrupt in ring-0).
6448 *
6449 * What should be important for the rest of the VBox code, is that the P bit is
6450 * cleared. Some of the other VBox code recognizes the unusable bit, but
6451 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6452 * safe side here, we'll strip off P and other bits we don't care about. If
6453 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6454 *
6455 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6456 */
6457 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6458 {
6459 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6460
6461 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6462 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6463 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6464
6465 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6466#ifdef DEBUG_bird
6467 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6468 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6469 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6470#endif
6471 }
6472 return VINF_SUCCESS;
6473}
6474
6475
6476#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6477# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6478 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6479 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6480#else
6481# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6482 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6483 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6484#endif
6485
6486
6487/**
6488 * Saves the guest segment registers from the current VMCS into the guest-CPU
6489 * context.
6490 *
6491 * @returns VBox status code.
6492 * @param pVCpu Pointer to the VMCPU.
6493 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6494 * out-of-sync. Make sure to update the required fields
6495 * before using them.
6496 *
6497 * @remarks No-long-jump zone!!!
6498 */
6499static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6500{
6501 /* Guest segment registers. */
6502 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6503 {
6504 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6505 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6506 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6507 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6508 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6509 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6510 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6511
6512 /* Restore segment attributes for real-on-v86 mode hack. */
6513 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6514 {
6515 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6516 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6517 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6518 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6519 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6520 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6521 }
6522 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6523 }
6524
6525 return VINF_SUCCESS;
6526}
6527
6528
6529/**
6530 * Saves the guest descriptor table registers and task register from the current
6531 * VMCS into the guest-CPU context.
6532 *
6533 * @returns VBox status code.
6534 * @param pVCpu Pointer to the VMCPU.
6535 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6536 * out-of-sync. Make sure to update the required fields
6537 * before using them.
6538 *
6539 * @remarks No-long-jump zone!!!
6540 */
6541static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6542{
6543 int rc = VINF_SUCCESS;
6544
6545 /* Guest LDTR. */
6546 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6547 {
6548 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6549 AssertRCReturn(rc, rc);
6550 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6551 }
6552
6553 /* Guest GDTR. */
6554 uint64_t u64Val = 0;
6555 uint32_t u32Val = 0;
6556 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6557 {
6558 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6559 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6560 pMixedCtx->gdtr.pGdt = u64Val;
6561 pMixedCtx->gdtr.cbGdt = u32Val;
6562 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6563 }
6564
6565 /* Guest IDTR. */
6566 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6567 {
6568 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6569 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6570 pMixedCtx->idtr.pIdt = u64Val;
6571 pMixedCtx->idtr.cbIdt = u32Val;
6572 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6573 }
6574
6575 /* Guest TR. */
6576 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6577 {
6578 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6579 AssertRCReturn(rc, rc);
6580
6581 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6582 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6583 {
6584 rc = VMXLOCAL_READ_SEG(TR, tr);
6585 AssertRCReturn(rc, rc);
6586 }
6587 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6588 }
6589 return rc;
6590}
6591
6592#undef VMXLOCAL_READ_SEG
6593
6594
6595/**
6596 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6597 * context.
6598 *
6599 * @returns VBox status code.
6600 * @param pVCpu Pointer to the VMCPU.
6601 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6602 * out-of-sync. Make sure to update the required fields
6603 * before using them.
6604 *
6605 * @remarks No-long-jump zone!!!
6606 */
6607static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6608{
6609 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6610 {
6611 if (!pVCpu->hm.s.fUsingHyperDR7)
6612 {
6613 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6614 uint32_t u32Val;
6615 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6616 pMixedCtx->dr[7] = u32Val;
6617 }
6618
6619 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6620 }
6621 return VINF_SUCCESS;
6622}
6623
6624
6625/**
6626 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6627 *
6628 * @returns VBox status code.
6629 * @param pVCpu Pointer to the VMCPU.
6630 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6631 * out-of-sync. Make sure to update the required fields
6632 * before using them.
6633 *
6634 * @remarks No-long-jump zone!!!
6635 */
6636static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6637{
6638 NOREF(pMixedCtx);
6639
6640 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6641 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6642 return VINF_SUCCESS;
6643}
6644
6645
6646/**
6647 * Saves the entire guest state from the currently active VMCS into the
6648 * guest-CPU context. This essentially VMREADs all guest-data.
6649 *
6650 * @returns VBox status code.
6651 * @param pVCpu Pointer to the VMCPU.
6652 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6653 * out-of-sync. Make sure to update the required fields
6654 * before using them.
6655 */
6656static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6657{
6658 Assert(pVCpu);
6659 Assert(pMixedCtx);
6660
6661 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6662 return VINF_SUCCESS;
6663
6664 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6665 again on the ring-3 callback path, there is no real need to. */
6666 if (VMMRZCallRing3IsEnabled(pVCpu))
6667 VMMR0LogFlushDisable(pVCpu);
6668 else
6669 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6670 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6671
6672 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6673 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6674
6675 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6676 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6677
6678 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6679 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6680
6681 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6682 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6683
6684 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6685 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6686
6687 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6688 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6689
6690 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6691 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6692
6693 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6694 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6695
6696 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6697 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6698
6699 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6700 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6701
6702 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6703 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6704
6705 if (VMMRZCallRing3IsEnabled(pVCpu))
6706 VMMR0LogFlushEnable(pVCpu);
6707
6708 return rc;
6709}
6710
6711
6712/**
6713 * Check per-VM and per-VCPU force flag actions that require us to go back to
6714 * ring-3 for one reason or another.
6715 *
6716 * @returns VBox status code (information status code included).
6717 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6718 * ring-3.
6719 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6720 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6721 * interrupts)
6722 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6723 * all EMTs to be in ring-3.
6724 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6725 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6726 * to the EM loop.
6727 *
6728 * @param pVM Pointer to the VM.
6729 * @param pVCpu Pointer to the VMCPU.
6730 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6731 * out-of-sync. Make sure to update the required fields
6732 * before using them.
6733 */
6734static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6735{
6736 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6737
6738 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6739 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6740 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6741 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6742 {
6743 /* We need the control registers now, make sure the guest-CPU context is updated. */
6744 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6745 AssertRCReturn(rc3, rc3);
6746
6747 /* Pending HM CR3 sync. */
6748 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6749 {
6750 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6751 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6752 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6753 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6754 }
6755
6756 /* Pending HM PAE PDPEs. */
6757 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6758 {
6759 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6760 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6761 }
6762
6763 /* Pending PGM C3 sync. */
6764 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6765 {
6766 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6767 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6768 if (rc2 != VINF_SUCCESS)
6769 {
6770 AssertRC(rc2);
6771 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6772 return rc2;
6773 }
6774 }
6775
6776 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6777 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6778 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6779 {
6780 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6781 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6782 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6783 return rc2;
6784 }
6785
6786 /* Pending VM request packets, such as hardware interrupts. */
6787 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6788 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6789 {
6790 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6791 return VINF_EM_PENDING_REQUEST;
6792 }
6793
6794 /* Pending PGM pool flushes. */
6795 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6796 {
6797 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6798 return VINF_PGM_POOL_FLUSH_PENDING;
6799 }
6800
6801 /* Pending DMA requests. */
6802 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6803 {
6804 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6805 return VINF_EM_RAW_TO_R3;
6806 }
6807 }
6808
6809 return VINF_SUCCESS;
6810}
6811
6812
6813/**
6814 * Converts any TRPM trap into a pending HM event. This is typically used when
6815 * entering from ring-3 (not longjmp returns).
6816 *
6817 * @param pVCpu Pointer to the VMCPU.
6818 */
6819static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6820{
6821 Assert(TRPMHasTrap(pVCpu));
6822 Assert(!pVCpu->hm.s.Event.fPending);
6823
6824 uint8_t uVector;
6825 TRPMEVENT enmTrpmEvent;
6826 RTGCUINT uErrCode;
6827 RTGCUINTPTR GCPtrFaultAddress;
6828 uint8_t cbInstr;
6829
6830 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6831 AssertRC(rc);
6832
6833 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6834 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6835 if (enmTrpmEvent == TRPM_TRAP)
6836 {
6837 switch (uVector)
6838 {
6839 case X86_XCPT_NMI:
6840 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6841 break;
6842
6843 case X86_XCPT_BP:
6844 case X86_XCPT_OF:
6845 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6846 break;
6847
6848 case X86_XCPT_PF:
6849 case X86_XCPT_DF:
6850 case X86_XCPT_TS:
6851 case X86_XCPT_NP:
6852 case X86_XCPT_SS:
6853 case X86_XCPT_GP:
6854 case X86_XCPT_AC:
6855 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6856 /* no break! */
6857 default:
6858 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6859 break;
6860 }
6861 }
6862 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6863 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6864 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6865 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6866 else
6867 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6868
6869 rc = TRPMResetTrap(pVCpu);
6870 AssertRC(rc);
6871 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6872 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6873
6874 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6875 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6876}
6877
6878
6879/**
6880 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6881 * VT-x to execute any instruction.
6882 *
6883 * @param pvCpu Pointer to the VMCPU.
6884 */
6885static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6886{
6887 Assert(pVCpu->hm.s.Event.fPending);
6888
6889 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6890 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6891 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6892 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6893
6894 /* If a trap was already pending, we did something wrong! */
6895 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6896
6897 TRPMEVENT enmTrapType;
6898 switch (uVectorType)
6899 {
6900 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6901 enmTrapType = TRPM_HARDWARE_INT;
6902 break;
6903
6904 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6905 enmTrapType = TRPM_SOFTWARE_INT;
6906 break;
6907
6908 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6909 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6910 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6911 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6912 enmTrapType = TRPM_TRAP;
6913 break;
6914
6915 default:
6916 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6917 enmTrapType = TRPM_32BIT_HACK;
6918 break;
6919 }
6920
6921 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6922
6923 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6924 AssertRC(rc);
6925
6926 if (fErrorCodeValid)
6927 TRPMSetErrorCode(pVCpu, uErrorCode);
6928
6929 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6930 && uVector == X86_XCPT_PF)
6931 {
6932 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6933 }
6934 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6935 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6936 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6937 {
6938 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6939 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6940 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6941 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6942 }
6943 pVCpu->hm.s.Event.fPending = false;
6944}
6945
6946
6947/**
6948 * Does the necessary state syncing before returning to ring-3 for any reason
6949 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6950 *
6951 * @returns VBox status code.
6952 * @param pVM Pointer to the VM.
6953 * @param pVCpu Pointer to the VMCPU.
6954 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6955 * be out-of-sync. Make sure to update the required
6956 * fields before using them.
6957 * @param fSaveGuestState Whether to save the guest state or not.
6958 *
6959 * @remarks No-long-jmp zone!!!
6960 */
6961static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6962{
6963 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6964 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6965
6966 RTCPUID idCpu = RTMpCpuId();
6967 Log4Func(("HostCpuId=%u\n", idCpu));
6968
6969 /*
6970 * !!! IMPORTANT !!!
6971 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6972 */
6973
6974 /* Save the guest state if necessary. */
6975 if ( fSaveGuestState
6976 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6977 {
6978 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6979 AssertRCReturn(rc, rc);
6980 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6981 }
6982
6983 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6984 if (CPUMIsGuestFPUStateActive(pVCpu))
6985 {
6986 /* We shouldn't reload CR0 without saving it first. */
6987 if (!fSaveGuestState)
6988 {
6989 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6990 AssertRCReturn(rc, rc);
6991 }
6992 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6993 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6994 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6995 }
6996
6997 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6998#ifdef VBOX_STRICT
6999 if (CPUMIsHyperDebugStateActive(pVCpu))
7000 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7001#endif
7002 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7003 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7004 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7005 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7006
7007#if HC_ARCH_BITS == 64
7008 /* Restore host-state bits that VT-x only restores partially. */
7009 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7010 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7011 {
7012 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7013 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7014 }
7015 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7016#endif
7017
7018#if HC_ARCH_BITS == 64
7019 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7020 if ( pVM->hm.s.fAllow64BitGuests
7021 && pVCpu->hm.s.vmx.fLazyMsrs)
7022 {
7023 /* We shouldn't reload the guest MSRs without saving it first. */
7024 if (!fSaveGuestState)
7025 {
7026 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7027 AssertRCReturn(rc, rc);
7028 }
7029 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7030 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7031 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7032 }
7033#endif
7034
7035 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7036 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7037
7038 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7039 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7040 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7041 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7042 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7043 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7044 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7045 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7046
7047 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7048
7049 /** @todo This partially defeats the purpose of having preemption hooks.
7050 * The problem is, deregistering the hooks should be moved to a place that
7051 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7052 * context.
7053 */
7054 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7055 {
7056 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7057 AssertRCReturn(rc, rc);
7058
7059 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7060 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7061 }
7062 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7063 NOREF(idCpu);
7064
7065 return VINF_SUCCESS;
7066}
7067
7068
7069/**
7070 * Leaves the VT-x session.
7071 *
7072 * @returns VBox status code.
7073 * @param pVM Pointer to the VM.
7074 * @param pVCpu Pointer to the VMCPU.
7075 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7076 * out-of-sync. Make sure to update the required fields
7077 * before using them.
7078 *
7079 * @remarks No-long-jmp zone!!!
7080 */
7081DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7082{
7083 HM_DISABLE_PREEMPT_IF_NEEDED();
7084 HMVMX_ASSERT_CPU_SAFE();
7085 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7086 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7087
7088 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7089 and done this from the VMXR0ThreadCtxCallback(). */
7090 if (!pVCpu->hm.s.fLeaveDone)
7091 {
7092 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7093 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7094 pVCpu->hm.s.fLeaveDone = true;
7095 }
7096 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7097
7098 /*
7099 * !!! IMPORTANT !!!
7100 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7101 */
7102
7103 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7104 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7105 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7106 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7107 VMMR0ThreadCtxHooksDeregister(pVCpu);
7108
7109 /* Leave HM context. This takes care of local init (term). */
7110 int rc = HMR0LeaveCpu(pVCpu);
7111
7112 HM_RESTORE_PREEMPT_IF_NEEDED();
7113
7114 return rc;
7115}
7116
7117
7118/**
7119 * Does the necessary state syncing before doing a longjmp to ring-3.
7120 *
7121 * @returns VBox status code.
7122 * @param pVM Pointer to the VM.
7123 * @param pVCpu Pointer to the VMCPU.
7124 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7125 * out-of-sync. Make sure to update the required fields
7126 * before using them.
7127 *
7128 * @remarks No-long-jmp zone!!!
7129 */
7130DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7131{
7132 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7133}
7134
7135
7136/**
7137 * Take necessary actions before going back to ring-3.
7138 *
7139 * An action requires us to go back to ring-3. This function does the necessary
7140 * steps before we can safely return to ring-3. This is not the same as longjmps
7141 * to ring-3, this is voluntary and prepares the guest so it may continue
7142 * executing outside HM (recompiler/IEM).
7143 *
7144 * @returns VBox status code.
7145 * @param pVM Pointer to the VM.
7146 * @param pVCpu Pointer to the VMCPU.
7147 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7148 * out-of-sync. Make sure to update the required fields
7149 * before using them.
7150 * @param rcExit The reason for exiting to ring-3. Can be
7151 * VINF_VMM_UNKNOWN_RING3_CALL.
7152 */
7153static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7154{
7155 Assert(pVM);
7156 Assert(pVCpu);
7157 Assert(pMixedCtx);
7158 HMVMX_ASSERT_PREEMPT_SAFE();
7159
7160 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7161 {
7162 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7163 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7164 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7165 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7166 }
7167
7168 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7169 VMMRZCallRing3Disable(pVCpu);
7170 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7171
7172 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7173 if (pVCpu->hm.s.Event.fPending)
7174 {
7175 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7176 Assert(!pVCpu->hm.s.Event.fPending);
7177 }
7178
7179 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7180 and if we're injecting an event we should have a TRPM trap pending. */
7181 Assert(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu));
7182 Assert(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu));
7183
7184 /* Save guest state and restore host state bits. */
7185 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7186 AssertRCReturn(rc, rc);
7187 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7188 /* Thread-context hooks are unregistered at this point!!! */
7189
7190 /* Sync recompiler state. */
7191 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7192 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7193 | CPUM_CHANGED_LDTR
7194 | CPUM_CHANGED_GDTR
7195 | CPUM_CHANGED_IDTR
7196 | CPUM_CHANGED_TR
7197 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7198 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7199 if ( pVM->hm.s.fNestedPaging
7200 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7201 {
7202 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7203 }
7204
7205 Assert(!pVCpu->hm.s.fClearTrapFlag);
7206
7207 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7208 if (rcExit != VINF_EM_RAW_INTERRUPT)
7209 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7210
7211 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7212
7213 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7214 VMMRZCallRing3RemoveNotification(pVCpu);
7215 VMMRZCallRing3Enable(pVCpu);
7216
7217 return rc;
7218}
7219
7220
7221/**
7222 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7223 * longjump to ring-3 and possibly get preempted.
7224 *
7225 * @returns VBox status code.
7226 * @param pVCpu Pointer to the VMCPU.
7227 * @param enmOperation The operation causing the ring-3 longjump.
7228 * @param pvUser Opaque pointer to the guest-CPU context. The data
7229 * may be out-of-sync. Make sure to update the required
7230 * fields before using them.
7231 */
7232DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7233{
7234 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7235 {
7236 /*
7237 * !!! IMPORTANT !!!
7238 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7239 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7240 */
7241 VMMRZCallRing3RemoveNotification(pVCpu);
7242 VMMRZCallRing3Disable(pVCpu);
7243 HM_DISABLE_PREEMPT_IF_NEEDED();
7244
7245 PVM pVM = pVCpu->CTX_SUFF(pVM);
7246 if (CPUMIsGuestFPUStateActive(pVCpu))
7247 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7248
7249 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7250
7251#if HC_ARCH_BITS == 64
7252 /* Restore host-state bits that VT-x only restores partially. */
7253 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7254 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7255 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7256 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7257
7258 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7259 if ( pVM->hm.s.fAllow64BitGuests
7260 && pVCpu->hm.s.vmx.fLazyMsrs)
7261 {
7262 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7263 }
7264#endif
7265 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7266 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7267 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7268 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7269 {
7270 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7271 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7272 }
7273
7274 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7275 VMMR0ThreadCtxHooksDeregister(pVCpu);
7276
7277 HMR0LeaveCpu(pVCpu);
7278 HM_RESTORE_PREEMPT_IF_NEEDED();
7279 return VINF_SUCCESS;
7280 }
7281
7282 Assert(pVCpu);
7283 Assert(pvUser);
7284 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7285 HMVMX_ASSERT_PREEMPT_SAFE();
7286
7287 VMMRZCallRing3Disable(pVCpu);
7288 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7289
7290 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7291 enmOperation));
7292
7293 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7294 AssertRCReturn(rc, rc);
7295
7296 VMMRZCallRing3Enable(pVCpu);
7297 return VINF_SUCCESS;
7298}
7299
7300
7301/**
7302 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7303 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7304 *
7305 * @param pVCpu Pointer to the VMCPU.
7306 */
7307DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7308{
7309 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7310 {
7311 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7312 {
7313 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7314 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7315 AssertRC(rc);
7316 Log4(("Setup interrupt-window exiting\n"));
7317 }
7318 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7319}
7320
7321
7322/**
7323 * Clears the interrupt-window exiting control in the VMCS.
7324 *
7325 * @param pVCpu Pointer to the VMCPU.
7326 */
7327DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7328{
7329 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7330 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7331 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7332 AssertRC(rc);
7333 Log4(("Cleared interrupt-window exiting\n"));
7334}
7335
7336
7337/**
7338 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7339 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7340 *
7341 * @param pVCpu Pointer to the VMCPU.
7342 */
7343DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7344{
7345 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7346 {
7347 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7348 {
7349 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7350 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7351 AssertRC(rc);
7352 Log4(("Setup NMI-window exiting\n"));
7353 }
7354 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7355}
7356
7357
7358/**
7359 * Clears the NMI-window exiting control in the VMCS.
7360 *
7361 * @param pVCpu Pointer to the VMCPU.
7362 */
7363DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7364{
7365 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7366 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7367 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7368 AssertRC(rc);
7369 Log4(("Cleared NMI-window exiting\n"));
7370}
7371
7372
7373/**
7374 * Evaluates the event to be delivered to the guest and sets it as the pending
7375 * event.
7376 *
7377 * @param pVCpu Pointer to the VMCPU.
7378 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7379 * out-of-sync. Make sure to update the required fields
7380 * before using them.
7381 */
7382static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7383{
7384 Assert(!pVCpu->hm.s.Event.fPending);
7385
7386 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7387 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7388 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7389 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7390 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7391
7392 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7393 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7394 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7395 Assert(!TRPMHasTrap(pVCpu));
7396
7397 /*
7398 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7399 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7400 */
7401 /** @todo SMI. SMIs take priority over NMIs. */
7402 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7403 {
7404 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7405 if ( !fBlockNmi
7406 && !fBlockSti
7407 && !fBlockMovSS)
7408 {
7409 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7410 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7411 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7412
7413 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7414 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7415 }
7416 else
7417 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7418 }
7419 /*
7420 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7421 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7422 */
7423 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7424 && !pVCpu->hm.s.fSingleInstruction)
7425 {
7426 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7427 AssertRC(rc);
7428 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7429 if ( !fBlockInt
7430 && !fBlockSti
7431 && !fBlockMovSS)
7432 {
7433 uint8_t u8Interrupt;
7434 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7435 if (RT_SUCCESS(rc))
7436 {
7437 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7438 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7439 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7440
7441 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7442 }
7443 else
7444 {
7445 /** @todo Does this actually happen? If not turn it into an assertion. */
7446 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7447 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7448 }
7449 }
7450 else
7451 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7452 }
7453}
7454
7455
7456/**
7457 * Sets a pending-debug exception to be delivered to the guest if the guest is
7458 * single-stepping.
7459 *
7460 * @param pVCpu Pointer to the VMCPU.
7461 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7462 * out-of-sync. Make sure to update the required fields
7463 * before using them.
7464 */
7465DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7466{
7467 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7468 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7469 {
7470 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7471 AssertRC(rc);
7472 }
7473}
7474
7475
7476/**
7477 * Injects any pending events into the guest if the guest is in a state to
7478 * receive them.
7479 *
7480 * @returns VBox status code (informational status codes included).
7481 * @param pVCpu Pointer to the VMCPU.
7482 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7483 * out-of-sync. Make sure to update the required fields
7484 * before using them.
7485 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7486 * return VINF_EM_DBG_STEPPED if the event was
7487 * dispatched directly.
7488 */
7489static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7490{
7491 HMVMX_ASSERT_PREEMPT_SAFE();
7492 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7493
7494 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7495 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7496 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7497 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7498
7499 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7500 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7501 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7502 Assert(!TRPMHasTrap(pVCpu));
7503
7504 int rc = VINF_SUCCESS;
7505 if (pVCpu->hm.s.Event.fPending)
7506 {
7507 /*
7508 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7509 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7510 * ended up enabling interrupts outside VT-x.
7511 */
7512 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7513 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7514 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7515 {
7516 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7517 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7518 }
7519
7520#ifdef VBOX_STRICT
7521 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7522 {
7523 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7524 Assert(!fBlockInt);
7525 Assert(!fBlockSti);
7526 Assert(!fBlockMovSS);
7527 }
7528 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7529 {
7530 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7531 Assert(!fBlockSti);
7532 Assert(!fBlockMovSS);
7533 Assert(!fBlockNmi);
7534 }
7535#endif
7536 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7537 (uint8_t)uIntType));
7538 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7539 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7540 AssertRCReturn(rc, rc);
7541
7542 /* Update the interruptibility-state as it could have been changed by
7543 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7544 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7545 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7546
7547#ifdef VBOX_WITH_STATISTICS
7548 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7549 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7550 else
7551 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7552#endif
7553 }
7554
7555 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7556 if ( fBlockSti
7557 || fBlockMovSS)
7558 {
7559 if ( !pVCpu->hm.s.fSingleInstruction
7560 && !DBGFIsStepping(pVCpu))
7561 {
7562 /*
7563 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7564 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7565 * See Intel spec. 27.3.4 "Saving Non-Register State".
7566 */
7567 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7568 AssertRCReturn(rc2, rc2);
7569 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7570 }
7571 else if (pMixedCtx->eflags.Bits.u1TF)
7572 {
7573 /*
7574 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7575 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7576 */
7577 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7578 uIntrState = 0;
7579 }
7580 }
7581
7582 /*
7583 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7584 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7585 */
7586 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7587 AssertRC(rc2);
7588
7589 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7590 NOREF(fBlockMovSS); NOREF(fBlockSti);
7591 return rc;
7592}
7593
7594
7595/**
7596 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7597 *
7598 * @param pVCpu Pointer to the VMCPU.
7599 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7600 * out-of-sync. Make sure to update the required fields
7601 * before using them.
7602 */
7603DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7604{
7605 NOREF(pMixedCtx);
7606 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7607 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7608}
7609
7610
7611/**
7612 * Injects a double-fault (#DF) exception into the VM.
7613 *
7614 * @returns VBox status code (informational status code included).
7615 * @param pVCpu Pointer to the VMCPU.
7616 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7617 * out-of-sync. Make sure to update the required fields
7618 * before using them.
7619 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7620 * and should return VINF_EM_DBG_STEPPED if the event
7621 * is injected directly (register modified by us, not
7622 * by hardware on VM-entry).
7623 * @param puIntrState Pointer to the current guest interruptibility-state.
7624 * This interruptibility-state will be updated if
7625 * necessary. This cannot not be NULL.
7626 */
7627DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7628{
7629 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7630 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7631 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7632 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7633 fStepping, puIntrState);
7634}
7635
7636
7637/**
7638 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7639 *
7640 * @param pVCpu Pointer to the VMCPU.
7641 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7642 * out-of-sync. Make sure to update the required fields
7643 * before using them.
7644 */
7645DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7646{
7647 NOREF(pMixedCtx);
7648 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7649 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7650 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7651}
7652
7653
7654/**
7655 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7656 *
7657 * @param pVCpu Pointer to the VMCPU.
7658 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7659 * out-of-sync. Make sure to update the required fields
7660 * before using them.
7661 * @param cbInstr The value of RIP that is to be pushed on the guest
7662 * stack.
7663 */
7664DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7665{
7666 NOREF(pMixedCtx);
7667 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7668 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7669 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7670}
7671
7672
7673/**
7674 * Injects a general-protection (#GP) fault into the VM.
7675 *
7676 * @returns VBox status code (informational status code included).
7677 * @param pVCpu Pointer to the VMCPU.
7678 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7679 * out-of-sync. Make sure to update the required fields
7680 * before using them.
7681 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7682 * mode, i.e. in real-mode it's not valid).
7683 * @param u32ErrorCode The error code associated with the #GP.
7684 * @param fStepping Whether we're running in
7685 * hmR0VmxRunGuestCodeStep() and should return
7686 * VINF_EM_DBG_STEPPED if the event is injected
7687 * directly (register modified by us, not by
7688 * hardware on VM-entry).
7689 * @param puIntrState Pointer to the current guest interruptibility-state.
7690 * This interruptibility-state will be updated if
7691 * necessary. This cannot not be NULL.
7692 */
7693DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7694 bool fStepping, uint32_t *puIntrState)
7695{
7696 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7697 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7698 if (fErrorCodeValid)
7699 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7700 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7701 fStepping, puIntrState);
7702}
7703
7704
7705/**
7706 * Sets a general-protection (#GP) exception as pending-for-injection into the
7707 * VM.
7708 *
7709 * @param pVCpu Pointer to the VMCPU.
7710 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7711 * out-of-sync. Make sure to update the required fields
7712 * before using them.
7713 * @param u32ErrorCode The error code associated with the #GP.
7714 */
7715DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7716{
7717 NOREF(pMixedCtx);
7718 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7719 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7720 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7721 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7722}
7723
7724
7725/**
7726 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7727 *
7728 * @param pVCpu Pointer to the VMCPU.
7729 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7730 * out-of-sync. Make sure to update the required fields
7731 * before using them.
7732 * @param uVector The software interrupt vector number.
7733 * @param cbInstr The value of RIP that is to be pushed on the guest
7734 * stack.
7735 */
7736DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7737{
7738 NOREF(pMixedCtx);
7739 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7740 if ( uVector == X86_XCPT_BP
7741 || uVector == X86_XCPT_OF)
7742 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7743 else
7744 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7745 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7746}
7747
7748
7749/**
7750 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7751 * stack.
7752 *
7753 * @returns VBox status code (information status code included).
7754 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7755 * @param pVM Pointer to the VM.
7756 * @param pMixedCtx Pointer to the guest-CPU context.
7757 * @param uValue The value to push to the guest stack.
7758 */
7759DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7760{
7761 /*
7762 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7763 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7764 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7765 */
7766 if (pMixedCtx->sp == 1)
7767 return VINF_EM_RESET;
7768 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7769 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7770 AssertRCReturn(rc, rc);
7771 return rc;
7772}
7773
7774
7775/**
7776 * Injects an event into the guest upon VM-entry by updating the relevant fields
7777 * in the VM-entry area in the VMCS.
7778 *
7779 * @returns VBox status code (informational error codes included).
7780 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7781 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7782 *
7783 * @param pVCpu Pointer to the VMCPU.
7784 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7785 * be out-of-sync. Make sure to update the required
7786 * fields before using them.
7787 * @param u64IntInfo The VM-entry interruption-information field.
7788 * @param cbInstr The VM-entry instruction length in bytes (for
7789 * software interrupts, exceptions and privileged
7790 * software exceptions).
7791 * @param u32ErrCode The VM-entry exception error code.
7792 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7793 * @param puIntrState Pointer to the current guest interruptibility-state.
7794 * This interruptibility-state will be updated if
7795 * necessary. This cannot not be NULL.
7796 * @param fStepping Whether we're running in
7797 * hmR0VmxRunGuestCodeStep() and should return
7798 * VINF_EM_DBG_STEPPED if the event is injected
7799 * directly (register modified by us, not by
7800 * hardware on VM-entry).
7801 *
7802 * @remarks Requires CR0!
7803 * @remarks No-long-jump zone!!!
7804 */
7805static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7806 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7807{
7808 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7809 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7810 Assert(puIntrState);
7811 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7812
7813 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7814 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7815
7816#ifdef VBOX_STRICT
7817 /* Validate the error-code-valid bit for hardware exceptions. */
7818 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7819 {
7820 switch (uVector)
7821 {
7822 case X86_XCPT_PF:
7823 case X86_XCPT_DF:
7824 case X86_XCPT_TS:
7825 case X86_XCPT_NP:
7826 case X86_XCPT_SS:
7827 case X86_XCPT_GP:
7828 case X86_XCPT_AC:
7829 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7830 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7831 /* fallthru */
7832 default:
7833 break;
7834 }
7835 }
7836#endif
7837
7838 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7839 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7840 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7841
7842 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7843
7844 /* We require CR0 to check if the guest is in real-mode. */
7845 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7846 AssertRCReturn(rc, rc);
7847
7848 /*
7849 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7850 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7851 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7852 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7853 */
7854 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7855 {
7856 PVM pVM = pVCpu->CTX_SUFF(pVM);
7857 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7858 {
7859 Assert(PDMVmmDevHeapIsEnabled(pVM));
7860 Assert(pVM->hm.s.vmx.pRealModeTSS);
7861
7862 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7863 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7864 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7865 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7866 AssertRCReturn(rc, rc);
7867 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7868
7869 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7870 size_t const cbIdtEntry = sizeof(X86IDTR16);
7871 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7872 {
7873 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7874 if (uVector == X86_XCPT_DF)
7875 return VINF_EM_RESET;
7876
7877 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7878 if (uVector == X86_XCPT_GP)
7879 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7880
7881 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7882 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7883 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7884 fStepping, puIntrState);
7885 }
7886
7887 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7888 uint16_t uGuestIp = pMixedCtx->ip;
7889 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7890 {
7891 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7892 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7893 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7894 }
7895 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7896 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7897
7898 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7899 X86IDTR16 IdtEntry;
7900 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7901 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7902 AssertRCReturn(rc, rc);
7903
7904 /* Construct the stack frame for the interrupt/exception handler. */
7905 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7906 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7907 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7908 AssertRCReturn(rc, rc);
7909
7910 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7911 if (rc == VINF_SUCCESS)
7912 {
7913 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7914 pMixedCtx->rip = IdtEntry.offSel;
7915 pMixedCtx->cs.Sel = IdtEntry.uSel;
7916 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7917 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7918 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7919 && uVector == X86_XCPT_PF)
7920 pMixedCtx->cr2 = GCPtrFaultAddress;
7921
7922 /* If any other guest-state bits are changed here, make sure to update
7923 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7924 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7925 | HM_CHANGED_GUEST_RIP
7926 | HM_CHANGED_GUEST_RFLAGS
7927 | HM_CHANGED_GUEST_RSP);
7928
7929 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7930 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7931 {
7932 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7933 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7934 Log4(("Clearing inhibition due to STI.\n"));
7935 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7936 }
7937 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7938 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7939
7940 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7941 it, if we are returning to ring-3 before executing guest code. */
7942 pVCpu->hm.s.Event.fPending = false;
7943
7944 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7945 if (fStepping)
7946 rc = VINF_EM_DBG_STEPPED;
7947 }
7948 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7949 return rc;
7950 }
7951
7952 /*
7953 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7954 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7955 */
7956 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7957 }
7958
7959 /* Validate. */
7960 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7961 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7962 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7963
7964 /* Inject. */
7965 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7966 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7967 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7968 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7969
7970 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7971 && uVector == X86_XCPT_PF)
7972 pMixedCtx->cr2 = GCPtrFaultAddress;
7973
7974 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7975 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7976
7977 AssertRCReturn(rc, rc);
7978 return rc;
7979}
7980
7981
7982/**
7983 * Clears the interrupt-window exiting control in the VMCS and if necessary
7984 * clears the current event in the VMCS as well.
7985 *
7986 * @returns VBox status code.
7987 * @param pVCpu Pointer to the VMCPU.
7988 *
7989 * @remarks Use this function only to clear events that have not yet been
7990 * delivered to the guest but are injected in the VMCS!
7991 * @remarks No-long-jump zone!!!
7992 */
7993static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7994{
7995 int rc;
7996 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7997
7998 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7999 {
8000 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8001 Assert(!pVCpu->hm.s.Event.fPending);
8002 }
8003
8004 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8005 {
8006 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8007 Assert(!pVCpu->hm.s.Event.fPending);
8008 }
8009
8010 if (!pVCpu->hm.s.Event.fPending)
8011 return;
8012
8013#ifdef VBOX_STRICT
8014 uint32_t u32EntryInfo;
8015 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8016 AssertRC(rc);
8017 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8018#endif
8019
8020 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8021 AssertRC(rc);
8022
8023 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8024 AssertRC(rc);
8025
8026 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8027 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8028}
8029
8030
8031/**
8032 * Enters the VT-x session.
8033 *
8034 * @returns VBox status code.
8035 * @param pVM Pointer to the VM.
8036 * @param pVCpu Pointer to the VMCPU.
8037 * @param pCpu Pointer to the CPU info struct.
8038 */
8039VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8040{
8041 AssertPtr(pVM);
8042 AssertPtr(pVCpu);
8043 Assert(pVM->hm.s.vmx.fSupported);
8044 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8045 NOREF(pCpu); NOREF(pVM);
8046
8047 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8048 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8049
8050#ifdef VBOX_STRICT
8051 /* Make sure we're in VMX root mode. */
8052 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8053 if (!(u32HostCR4 & X86_CR4_VMXE))
8054 {
8055 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8056 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8057 }
8058#endif
8059
8060 /*
8061 * Load the VCPU's VMCS as the current (and active) one.
8062 */
8063 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8064 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8065 if (RT_FAILURE(rc))
8066 return rc;
8067
8068 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8069 pVCpu->hm.s.fLeaveDone = false;
8070 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8071
8072 return VINF_SUCCESS;
8073}
8074
8075
8076/**
8077 * The thread-context callback (only on platforms which support it).
8078 *
8079 * @param enmEvent The thread-context event.
8080 * @param pVCpu Pointer to the VMCPU.
8081 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8082 * @thread EMT(pVCpu)
8083 */
8084VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8085{
8086 NOREF(fGlobalInit);
8087
8088 switch (enmEvent)
8089 {
8090 case RTTHREADCTXEVENT_PREEMPTING:
8091 {
8092 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8093 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8094 VMCPU_ASSERT_EMT(pVCpu);
8095
8096 PVM pVM = pVCpu->CTX_SUFF(pVM);
8097 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8098
8099 /* No longjmps (logger flushes, locks) in this fragile context. */
8100 VMMRZCallRing3Disable(pVCpu);
8101 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8102
8103 /*
8104 * Restore host-state (FPU, debug etc.)
8105 */
8106 if (!pVCpu->hm.s.fLeaveDone)
8107 {
8108 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8109 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8110 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8111 pVCpu->hm.s.fLeaveDone = true;
8112 }
8113
8114 /* Leave HM context, takes care of local init (term). */
8115 int rc = HMR0LeaveCpu(pVCpu);
8116 AssertRC(rc); NOREF(rc);
8117
8118 /* Restore longjmp state. */
8119 VMMRZCallRing3Enable(pVCpu);
8120 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8121 break;
8122 }
8123
8124 case RTTHREADCTXEVENT_RESUMED:
8125 {
8126 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8127 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8128 VMCPU_ASSERT_EMT(pVCpu);
8129
8130 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8131 VMMRZCallRing3Disable(pVCpu);
8132 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8133
8134 /* Initialize the bare minimum state required for HM. This takes care of
8135 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8136 int rc = HMR0EnterCpu(pVCpu);
8137 AssertRC(rc);
8138 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8139
8140 /* Load the active VMCS as the current one. */
8141 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8142 {
8143 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8144 AssertRC(rc); NOREF(rc);
8145 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8146 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8147 }
8148 pVCpu->hm.s.fLeaveDone = false;
8149
8150 /* Restore longjmp state. */
8151 VMMRZCallRing3Enable(pVCpu);
8152 break;
8153 }
8154
8155 default:
8156 break;
8157 }
8158}
8159
8160
8161/**
8162 * Saves the host state in the VMCS host-state.
8163 * Sets up the VM-exit MSR-load area.
8164 *
8165 * The CPU state will be loaded from these fields on every successful VM-exit.
8166 *
8167 * @returns VBox status code.
8168 * @param pVM Pointer to the VM.
8169 * @param pVCpu Pointer to the VMCPU.
8170 *
8171 * @remarks No-long-jump zone!!!
8172 */
8173static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8174{
8175 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8176
8177 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8178 return VINF_SUCCESS;
8179
8180 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8181 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8182
8183 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8184 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8185
8186 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8187 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8188
8189 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8190 return rc;
8191}
8192
8193
8194/**
8195 * Saves the host state in the VMCS host-state.
8196 *
8197 * @returns VBox status code.
8198 * @param pVM Pointer to the VM.
8199 * @param pVCpu Pointer to the VMCPU.
8200 *
8201 * @remarks No-long-jump zone!!!
8202 */
8203VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8204{
8205 AssertPtr(pVM);
8206 AssertPtr(pVCpu);
8207
8208 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8209
8210 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8211 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8212 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8213 return hmR0VmxSaveHostState(pVM, pVCpu);
8214}
8215
8216
8217/**
8218 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8219 * loaded from these fields on every successful VM-entry.
8220 *
8221 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8222 * Sets up the VM-entry controls.
8223 * Sets up the appropriate VMX non-root function to execute guest code based on
8224 * the guest CPU mode.
8225 *
8226 * @returns VBox status code.
8227 * @param pVM Pointer to the VM.
8228 * @param pVCpu Pointer to the VMCPU.
8229 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8230 * out-of-sync. Make sure to update the required fields
8231 * before using them.
8232 *
8233 * @remarks No-long-jump zone!!!
8234 */
8235static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8236{
8237 AssertPtr(pVM);
8238 AssertPtr(pVCpu);
8239 AssertPtr(pMixedCtx);
8240 HMVMX_ASSERT_PREEMPT_SAFE();
8241
8242 VMMRZCallRing3Disable(pVCpu);
8243 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8244
8245 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8246
8247 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8248
8249 /* Determine real-on-v86 mode. */
8250 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8251 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8252 && CPUMIsGuestInRealModeEx(pMixedCtx))
8253 {
8254 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8255 }
8256
8257 /*
8258 * Load the guest-state into the VMCS.
8259 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8260 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8261 */
8262 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8263 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8264
8265 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8266 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8267 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8268
8269 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8270 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8271 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8272
8273 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8274 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8275
8276 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8277 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8278
8279 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8280 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8281 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8282
8283 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8284 determine we don't have to swap EFER after all. */
8285 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8286 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8287
8288 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8289 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8290
8291 /*
8292 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8293 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8294 */
8295 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8296 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8297
8298 /* Clear any unused and reserved bits. */
8299 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8300
8301 VMMRZCallRing3Enable(pVCpu);
8302
8303 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8304 return rc;
8305}
8306
8307
8308/**
8309 * Loads the state shared between the host and guest into the VMCS.
8310 *
8311 * @param pVM Pointer to the VM.
8312 * @param pVCpu Pointer to the VMCPU.
8313 * @param pCtx Pointer to the guest-CPU context.
8314 *
8315 * @remarks No-long-jump zone!!!
8316 */
8317static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8318{
8319 NOREF(pVM);
8320
8321 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8322 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8323
8324 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8325 {
8326 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8327 AssertRC(rc);
8328 }
8329
8330 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8331 {
8332 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8333 AssertRC(rc);
8334
8335 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8336 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8337 {
8338 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8339 AssertRC(rc);
8340 }
8341 }
8342
8343 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8344 {
8345#if HC_ARCH_BITS == 64
8346 if (pVM->hm.s.fAllow64BitGuests)
8347 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8348#endif
8349 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8350 }
8351
8352 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8353 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8354}
8355
8356
8357/**
8358 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8359 *
8360 * @param pVM Pointer to the VM.
8361 * @param pVCpu Pointer to the VMCPU.
8362 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8363 * out-of-sync. Make sure to update the required fields
8364 * before using them.
8365 */
8366DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8367{
8368 HMVMX_ASSERT_PREEMPT_SAFE();
8369
8370 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8371#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8372 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8373#endif
8374
8375 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8376 {
8377 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8378 AssertRC(rc);
8379 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8380 }
8381 else if (HMCPU_CF_VALUE(pVCpu))
8382 {
8383 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8384 AssertRC(rc);
8385 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8386 }
8387
8388 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8389 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8390 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8391 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8392}
8393
8394
8395/**
8396 * Does the preparations before executing guest code in VT-x.
8397 *
8398 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8399 * recompiler/IEM. We must be cautious what we do here regarding committing
8400 * guest-state information into the VMCS assuming we assuredly execute the
8401 * guest in VT-x mode.
8402 *
8403 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8404 * the common-state (TRPM/forceflags), we must undo those changes so that the
8405 * recompiler/IEM can (and should) use them when it resumes guest execution.
8406 * Otherwise such operations must be done when we can no longer exit to ring-3.
8407 *
8408 * @returns Strict VBox status code.
8409 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8410 * have been disabled.
8411 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8412 * double-fault into the guest.
8413 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8414 * dispatched directly.
8415 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8416 *
8417 * @param pVM Pointer to the VM.
8418 * @param pVCpu Pointer to the VMCPU.
8419 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8420 * out-of-sync. Make sure to update the required fields
8421 * before using them.
8422 * @param pVmxTransient Pointer to the VMX transient structure.
8423 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8424 * us ignore some of the reasons for returning to
8425 * ring-3, and return VINF_EM_DBG_STEPPED if event
8426 * dispatching took place.
8427 */
8428static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8429{
8430 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8431
8432#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8433 PGMRZDynMapFlushAutoSet(pVCpu);
8434#endif
8435
8436 /* Check force flag actions that might require us to go back to ring-3. */
8437 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8438 if (rc != VINF_SUCCESS)
8439 return rc;
8440
8441#ifndef IEM_VERIFICATION_MODE_FULL
8442 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8443 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8444 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8445 {
8446 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8447 RTGCPHYS GCPhysApicBase;
8448 GCPhysApicBase = pMixedCtx->msrApicBase;
8449 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8450
8451 /* Unalias any existing mapping. */
8452 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8453 AssertRCReturn(rc, rc);
8454
8455 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8456 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8457 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8458 AssertRCReturn(rc, rc);
8459
8460 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8461 }
8462#endif /* !IEM_VERIFICATION_MODE_FULL */
8463
8464 if (TRPMHasTrap(pVCpu))
8465 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8466 else if (!pVCpu->hm.s.Event.fPending)
8467 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8468
8469 /*
8470 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8471 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8472 */
8473 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8474 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8475 {
8476 Assert(rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
8477 return rc;
8478 }
8479
8480 /*
8481 * Load the guest state bits, we can handle longjmps/getting preempted here.
8482 *
8483 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8484 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8485 * Hence, this needs to be done -after- injection of events.
8486 */
8487 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8488
8489 /*
8490 * No longjmps to ring-3 from this point on!!!
8491 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8492 * This also disables flushing of the R0-logger instance (if any).
8493 */
8494 VMMRZCallRing3Disable(pVCpu);
8495
8496 /*
8497 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8498 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8499 *
8500 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8501 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8502 *
8503 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8504 * executing guest code.
8505 */
8506 pVmxTransient->uEflags = ASMIntDisableFlags();
8507 if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8508 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8509 && ( !fStepping /* Optimized for the non-stepping case, of course. */
8510 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8511 {
8512 hmR0VmxClearEventVmcs(pVCpu);
8513 ASMSetFlags(pVmxTransient->uEflags);
8514 VMMRZCallRing3Enable(pVCpu);
8515 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8516 return VINF_EM_RAW_TO_R3;
8517 }
8518
8519 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8520 {
8521 hmR0VmxClearEventVmcs(pVCpu);
8522 ASMSetFlags(pVmxTransient->uEflags);
8523 VMMRZCallRing3Enable(pVCpu);
8524 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8525 return VINF_EM_RAW_INTERRUPT;
8526 }
8527
8528 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8529 pVCpu->hm.s.Event.fPending = false;
8530
8531 return VINF_SUCCESS;
8532}
8533
8534
8535/**
8536 * Prepares to run guest code in VT-x and we've committed to doing so. This
8537 * means there is no backing out to ring-3 or anywhere else at this
8538 * point.
8539 *
8540 * @param pVM Pointer to the VM.
8541 * @param pVCpu Pointer to the VMCPU.
8542 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8543 * out-of-sync. Make sure to update the required fields
8544 * before using them.
8545 * @param pVmxTransient Pointer to the VMX transient structure.
8546 *
8547 * @remarks Called with preemption disabled.
8548 * @remarks No-long-jump zone!!!
8549 */
8550static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8551{
8552 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8553 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8554 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8555
8556 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8557 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8558
8559#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8560 if (!CPUMIsGuestFPUStateActive(pVCpu))
8561 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8562 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8563#endif
8564
8565 if ( pVCpu->hm.s.fUseGuestFpu
8566 && !CPUMIsGuestFPUStateActive(pVCpu))
8567 {
8568 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8569 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8570 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8571 }
8572
8573 /*
8574 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8575 */
8576 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8577 && pVCpu->hm.s.vmx.cMsrs > 0)
8578 {
8579 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8580 }
8581
8582 /*
8583 * Load the host state bits as we may've been preempted (only happens when
8584 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8585 */
8586 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8587 {
8588 /* This ASSUMES that pfnStartVM has been set up already. */
8589 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8590 AssertRC(rc);
8591 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8592 }
8593 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8594
8595 /*
8596 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8597 */
8598 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8599 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8600 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8601
8602 /* Store status of the shared guest-host state at the time of VM-entry. */
8603#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8604 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8605 {
8606 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8607 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8608 }
8609 else
8610#endif
8611 {
8612 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8613 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8614 }
8615 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8616
8617 /*
8618 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8619 */
8620 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8621 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8622
8623 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8624 RTCPUID idCurrentCpu = pCpu->idCpu;
8625 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8626 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8627 {
8628 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8629 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8630 }
8631
8632 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8633 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8634 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8635 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8636
8637 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8638
8639 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8640 to start executing. */
8641
8642 /*
8643 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8644 */
8645 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8646 {
8647 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8648 {
8649 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8650 AssertRC(rc2);
8651 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8652 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8653 true /* fUpdateHostMsr */);
8654 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8655 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8656 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8657 }
8658 else
8659 {
8660 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8661 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8662 }
8663 }
8664
8665#ifdef VBOX_STRICT
8666 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8667 hmR0VmxCheckHostEferMsr(pVCpu);
8668 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8669#endif
8670#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8671 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8672 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8673 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8674#endif
8675}
8676
8677
8678/**
8679 * Performs some essential restoration of state after running guest code in
8680 * VT-x.
8681 *
8682 * @param pVM Pointer to the VM.
8683 * @param pVCpu Pointer to the VMCPU.
8684 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8685 * out-of-sync. Make sure to update the required fields
8686 * before using them.
8687 * @param pVmxTransient Pointer to the VMX transient structure.
8688 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8689 *
8690 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8691 *
8692 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8693 * unconditionally when it is safe to do so.
8694 */
8695static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8696{
8697 NOREF(pVM);
8698
8699 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8700
8701 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8702 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8703 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8704 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8705 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8706 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8707
8708 /** @todo Last-seen-tick shouldn't be necessary when TM supports invariant
8709 * mode. */
8710 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8711 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8712
8713 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8714 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8715 Assert(!(ASMGetFlags() & X86_EFL_IF));
8716 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8717
8718#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8719 if (CPUMIsGuestFPUStateActive(pVCpu))
8720 {
8721 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8722 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8723 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8724 }
8725#endif
8726
8727#if HC_ARCH_BITS == 64
8728 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8729#endif
8730 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8731#ifdef VBOX_STRICT
8732 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8733#endif
8734 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8735 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8736
8737 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8738 uint32_t uExitReason;
8739 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8740 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8741 AssertRC(rc);
8742 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8743 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8744
8745 /* Update the VM-exit history array. */
8746 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8747
8748 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8749 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8750 {
8751 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8752 pVmxTransient->fVMEntryFailed));
8753 return;
8754 }
8755
8756 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8757 {
8758 /** @todo We can optimize this by only syncing with our force-flags when
8759 * really needed and keeping the VMCS state as it is for most
8760 * VM-exits. */
8761 /* Update the guest interruptibility-state from the VMCS. */
8762 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8763
8764#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8765 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8766 AssertRC(rc);
8767#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8768 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8769 AssertRC(rc);
8770#endif
8771
8772 /*
8773 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8774 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8775 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8776 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8777 */
8778 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8779 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8780 {
8781 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8782 AssertRC(rc);
8783 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8784 }
8785 }
8786}
8787
8788
8789/**
8790 * Runs the guest code using VT-x the normal way.
8791 *
8792 * @returns VBox status code.
8793 * @param pVM Pointer to the VM.
8794 * @param pVCpu Pointer to the VMCPU.
8795 * @param pCtx Pointer to the guest-CPU context.
8796 *
8797 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8798 */
8799static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8800{
8801 VMXTRANSIENT VmxTransient;
8802 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8803 int rc = VERR_INTERNAL_ERROR_5;
8804 uint32_t cLoops = 0;
8805
8806 for (;; cLoops++)
8807 {
8808 Assert(!HMR0SuspendPending());
8809 HMVMX_ASSERT_CPU_SAFE();
8810
8811 /* Preparatory work for running guest code, this may force us to return
8812 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8813 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8814 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8815 if (rc != VINF_SUCCESS)
8816 break;
8817
8818 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8819 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8820 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8821
8822 /* Restore any residual host-state and save any bits shared between host
8823 and guest into the guest-CPU state. Re-enables interrupts! */
8824 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8825
8826 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8827 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8828 {
8829 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8830 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8831 return rc;
8832 }
8833
8834 /* Handle the VM-exit. */
8835 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8837 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8838 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8839 HMVMX_START_EXIT_DISPATCH_PROF();
8840#ifdef HMVMX_USE_FUNCTION_TABLE
8841 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8842#else
8843 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8844#endif
8845 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8846 if (rc != VINF_SUCCESS)
8847 break;
8848 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8849 {
8850 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8851 rc = VINF_EM_RAW_INTERRUPT;
8852 break;
8853 }
8854 }
8855
8856 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8857 return rc;
8858}
8859
8860
8861/**
8862 * Single steps guest code using VT-x.
8863 *
8864 * @returns VBox status code.
8865 * @param pVM Pointer to the VM.
8866 * @param pVCpu Pointer to the VMCPU.
8867 * @param pCtx Pointer to the guest-CPU context.
8868 *
8869 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8870 */
8871static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8872{
8873 VMXTRANSIENT VmxTransient;
8874 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8875 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8876 uint32_t cLoops = 0;
8877 uint16_t uCsStart = pCtx->cs.Sel;
8878 uint64_t uRipStart = pCtx->rip;
8879
8880 for (;; cLoops++)
8881 {
8882 Assert(!HMR0SuspendPending());
8883 HMVMX_ASSERT_CPU_SAFE();
8884
8885 /* Preparatory work for running guest code, this may force us to return
8886 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8887 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8888 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8889 if (rcStrict != VINF_SUCCESS)
8890 break;
8891
8892 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8893 rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8894 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8895
8896 /* Restore any residual host-state and save any bits shared between host
8897 and guest into the guest-CPU state. Re-enables interrupts! */
8898 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8899
8900 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8901 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
8902 {
8903 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8904 hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
8905 return VBOXSTRICTRC_TODO(rcStrict);
8906 }
8907
8908 /* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8909 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8910 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8911 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8912 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8913 HMVMX_START_EXIT_DISPATCH_PROF();
8914 rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8915 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8916 if (rcStrict != VINF_SUCCESS)
8917 break;
8918 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8919 {
8920 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8921 rcStrict = VINF_EM_RAW_INTERRUPT;
8922 break;
8923 }
8924
8925 /*
8926 * Did the RIP change, if so, consider it a single step.
8927 * Otherwise, make sure one of the TFs gets set.
8928 */
8929 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8930 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8931 AssertRCReturn(rc2, rc2);
8932 if ( pCtx->rip != uRipStart
8933 || pCtx->cs.Sel != uCsStart)
8934 {
8935 rcStrict = VINF_EM_DBG_STEPPED;
8936 break;
8937 }
8938 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8939 }
8940
8941 /*
8942 * Clear the X86_EFL_TF if necessary.
8943 */
8944 if (pVCpu->hm.s.fClearTrapFlag)
8945 {
8946 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8947 AssertRCReturn(rc2, rc2);
8948 pVCpu->hm.s.fClearTrapFlag = false;
8949 pCtx->eflags.Bits.u1TF = 0;
8950 }
8951 /** @todo there seems to be issues with the resume flag when the monitor trap
8952 * flag is pending without being used. Seen early in bios init when
8953 * accessing APIC page in protected mode. */
8954
8955 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8956 return VBOXSTRICTRC_TODO(rcStrict);
8957}
8958
8959
8960/**
8961 * Runs the guest code using VT-x.
8962 *
8963 * @returns VBox status code.
8964 * @param pVM Pointer to the VM.
8965 * @param pVCpu Pointer to the VMCPU.
8966 * @param pCtx Pointer to the guest-CPU context.
8967 */
8968VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8969{
8970 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8971 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8972 HMVMX_ASSERT_PREEMPT_SAFE();
8973
8974 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8975
8976 int rc;
8977 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8978 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8979 else
8980 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8981
8982 if (rc == VERR_EM_INTERPRETER)
8983 rc = VINF_EM_RAW_EMULATE_INSTR;
8984 else if (rc == VINF_EM_RESET)
8985 rc = VINF_EM_TRIPLE_FAULT;
8986
8987 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8988 if (RT_FAILURE(rc2))
8989 {
8990 pVCpu->hm.s.u32HMError = rc;
8991 rc = rc2;
8992 }
8993 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8994 return rc;
8995}
8996
8997
8998#ifndef HMVMX_USE_FUNCTION_TABLE
8999DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9000{
9001#ifdef DEBUG_ramshankar
9002# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9003# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9004#endif
9005 int rc;
9006 switch (rcReason)
9007 {
9008 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9009 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9010 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9011 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9012 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9013 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9014 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9015 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9016 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9017 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9018 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9019 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9020 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9021 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9022 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9023 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9024 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9025 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9026 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9027 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9028 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9029 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9030 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9031 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9032 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9033 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9034 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9035 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9036 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9037 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9038 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9039 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9040 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9041 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9042
9043 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9044 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9045 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9046 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9047 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9048 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9049 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9050 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9051 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9052
9053 case VMX_EXIT_VMCLEAR:
9054 case VMX_EXIT_VMLAUNCH:
9055 case VMX_EXIT_VMPTRLD:
9056 case VMX_EXIT_VMPTRST:
9057 case VMX_EXIT_VMREAD:
9058 case VMX_EXIT_VMRESUME:
9059 case VMX_EXIT_VMWRITE:
9060 case VMX_EXIT_VMXOFF:
9061 case VMX_EXIT_VMXON:
9062 case VMX_EXIT_INVEPT:
9063 case VMX_EXIT_INVVPID:
9064 case VMX_EXIT_VMFUNC:
9065 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9066 break;
9067 default:
9068 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9069 break;
9070 }
9071 return rc;
9072}
9073#endif /* !HMVMX_USE_FUNCTION_TABLE */
9074
9075
9076/**
9077 * Single-stepping VM-exit filtering.
9078 *
9079 * This is preprocessing the exits and deciding whether we've gotten far enough
9080 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9081 * performed.
9082 *
9083 * @returns Strict VBox status code.
9084 * @param pVCpu The virtual CPU of the calling EMT.
9085 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9086 * out-of-sync. Make sure to update the required
9087 * fields before using them.
9088 * @param pVmxTransient Pointer to the VMX-transient structure.
9089 * @param uExitReason The VM-exit reason.
9090 */
9091DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9092 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9093{
9094 switch (uExitReason)
9095 {
9096 case VMX_EXIT_XCPT_OR_NMI:
9097 {
9098 /* Check for host NMI. */
9099 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9100 AssertRCReturn(rc2, rc2);
9101 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9102 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9103 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9104 /* fall thru */
9105 }
9106
9107 case VMX_EXIT_EPT_MISCONFIG:
9108 case VMX_EXIT_TRIPLE_FAULT:
9109 case VMX_EXIT_APIC_ACCESS:
9110 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9111 case VMX_EXIT_TASK_SWITCH:
9112
9113 /* Instruction specific VM-exits: */
9114 case VMX_EXIT_IO_INSTR:
9115 case VMX_EXIT_CPUID:
9116 case VMX_EXIT_RDTSC:
9117 case VMX_EXIT_RDTSCP:
9118 case VMX_EXIT_MOV_CRX:
9119 case VMX_EXIT_MWAIT:
9120 case VMX_EXIT_MONITOR:
9121 case VMX_EXIT_RDMSR:
9122 case VMX_EXIT_WRMSR:
9123 case VMX_EXIT_MOV_DRX:
9124 case VMX_EXIT_HLT:
9125 case VMX_EXIT_INVD:
9126 case VMX_EXIT_INVLPG:
9127 case VMX_EXIT_RSM:
9128 case VMX_EXIT_PAUSE:
9129 case VMX_EXIT_XDTR_ACCESS:
9130 case VMX_EXIT_TR_ACCESS:
9131 case VMX_EXIT_WBINVD:
9132 case VMX_EXIT_XSETBV:
9133 case VMX_EXIT_RDRAND:
9134 case VMX_EXIT_INVPCID:
9135 case VMX_EXIT_GETSEC:
9136 case VMX_EXIT_RDPMC:
9137 case VMX_EXIT_VMCALL:
9138 case VMX_EXIT_VMCLEAR:
9139 case VMX_EXIT_VMLAUNCH:
9140 case VMX_EXIT_VMPTRLD:
9141 case VMX_EXIT_VMPTRST:
9142 case VMX_EXIT_VMREAD:
9143 case VMX_EXIT_VMRESUME:
9144 case VMX_EXIT_VMWRITE:
9145 case VMX_EXIT_VMXOFF:
9146 case VMX_EXIT_VMXON:
9147 case VMX_EXIT_INVEPT:
9148 case VMX_EXIT_INVVPID:
9149 case VMX_EXIT_VMFUNC:
9150 {
9151 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9152 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9153 AssertRCReturn(rc2, rc2);
9154 if ( pMixedCtx->rip != uRipStart
9155 || pMixedCtx->cs.Sel != uCsStart)
9156 return VINF_EM_DBG_STEPPED;
9157 break;
9158 }
9159 }
9160
9161 /*
9162 * Normal processing.
9163 */
9164#ifdef HMVMX_USE_FUNCTION_TABLE
9165 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9166#else
9167 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9168#endif
9169}
9170
9171
9172#ifdef DEBUG
9173/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9174# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9175 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9176
9177# define HMVMX_ASSERT_PREEMPT_CPUID() \
9178 do \
9179 { \
9180 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9181 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9182 } while (0)
9183
9184# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9185 do { \
9186 AssertPtr(pVCpu); \
9187 AssertPtr(pMixedCtx); \
9188 AssertPtr(pVmxTransient); \
9189 Assert(pVmxTransient->fVMEntryFailed == false); \
9190 Assert(ASMIntAreEnabled()); \
9191 HMVMX_ASSERT_PREEMPT_SAFE(); \
9192 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9193 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9194 HMVMX_ASSERT_PREEMPT_SAFE(); \
9195 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9196 HMVMX_ASSERT_PREEMPT_CPUID(); \
9197 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9198 } while (0)
9199
9200# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9201 do { \
9202 Log4Func(("\n")); \
9203 } while (0)
9204#else /* Release builds */
9205# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9206 do { \
9207 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9208 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9209 } while (0)
9210# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9211#endif
9212
9213
9214/**
9215 * Advances the guest RIP after reading it from the VMCS.
9216 *
9217 * @returns VBox status code.
9218 * @param pVCpu Pointer to the VMCPU.
9219 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9220 * out-of-sync. Make sure to update the required fields
9221 * before using them.
9222 * @param pVmxTransient Pointer to the VMX transient structure.
9223 *
9224 * @remarks No-long-jump zone!!!
9225 */
9226DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9227{
9228 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9229 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9230 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9231 AssertRCReturn(rc, rc);
9232
9233 pMixedCtx->rip += pVmxTransient->cbInstr;
9234 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9235
9236 /*
9237 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9238 * pending debug exception field as it takes care of priority of events.
9239 *
9240 * See Intel spec. 32.2.1 "Debug Exceptions".
9241 */
9242 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9243
9244 return rc;
9245}
9246
9247
9248/**
9249 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9250 * and update error record fields accordingly.
9251 *
9252 * @return VMX_IGS_* return codes.
9253 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9254 * wrong with the guest state.
9255 *
9256 * @param pVM Pointer to the VM.
9257 * @param pVCpu Pointer to the VMCPU.
9258 * @param pCtx Pointer to the guest-CPU state.
9259 *
9260 * @remarks This function assumes our cache of the VMCS controls
9261 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9262 */
9263static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9264{
9265#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9266#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9267 uError = (err); \
9268 break; \
9269 } else do { } while (0)
9270
9271 int rc;
9272 uint32_t uError = VMX_IGS_ERROR;
9273 uint32_t u32Val;
9274 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9275
9276 do
9277 {
9278 /*
9279 * CR0.
9280 */
9281 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9282 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9283 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9284 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9285 if (fUnrestrictedGuest)
9286 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9287
9288 uint32_t u32GuestCR0;
9289 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9290 AssertRCBreak(rc);
9291 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9292 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9293 if ( !fUnrestrictedGuest
9294 && (u32GuestCR0 & X86_CR0_PG)
9295 && !(u32GuestCR0 & X86_CR0_PE))
9296 {
9297 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9298 }
9299
9300 /*
9301 * CR4.
9302 */
9303 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9304 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9305
9306 uint32_t u32GuestCR4;
9307 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9308 AssertRCBreak(rc);
9309 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9310 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9311
9312 /*
9313 * IA32_DEBUGCTL MSR.
9314 */
9315 uint64_t u64Val;
9316 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9317 AssertRCBreak(rc);
9318 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9319 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9320 {
9321 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9322 }
9323 uint64_t u64DebugCtlMsr = u64Val;
9324
9325#ifdef VBOX_STRICT
9326 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9327 AssertRCBreak(rc);
9328 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9329#endif
9330 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9331
9332 /*
9333 * RIP and RFLAGS.
9334 */
9335 uint32_t u32Eflags;
9336#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9337 if (HMVMX_IS_64BIT_HOST_MODE())
9338 {
9339 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9340 AssertRCBreak(rc);
9341 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9342 if ( !fLongModeGuest
9343 || !pCtx->cs.Attr.n.u1Long)
9344 {
9345 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9346 }
9347 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9348 * must be identical if the "IA-32e mode guest" VM-entry
9349 * control is 1 and CS.L is 1. No check applies if the
9350 * CPU supports 64 linear-address bits. */
9351
9352 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9353 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9354 AssertRCBreak(rc);
9355 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9356 VMX_IGS_RFLAGS_RESERVED);
9357 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9358 u32Eflags = u64Val;
9359 }
9360 else
9361#endif
9362 {
9363 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9364 AssertRCBreak(rc);
9365 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9366 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9367 }
9368
9369 if ( fLongModeGuest
9370 || ( fUnrestrictedGuest
9371 && !(u32GuestCR0 & X86_CR0_PE)))
9372 {
9373 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9374 }
9375
9376 uint32_t u32EntryInfo;
9377 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9378 AssertRCBreak(rc);
9379 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9380 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9381 {
9382 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9383 }
9384
9385 /*
9386 * 64-bit checks.
9387 */
9388#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9389 if (HMVMX_IS_64BIT_HOST_MODE())
9390 {
9391 if ( fLongModeGuest
9392 && !fUnrestrictedGuest)
9393 {
9394 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9395 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9396 }
9397
9398 if ( !fLongModeGuest
9399 && (u32GuestCR4 & X86_CR4_PCIDE))
9400 {
9401 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9402 }
9403
9404 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9405 * 51:32 beyond the processor's physical-address width are 0. */
9406
9407 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9408 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9409 {
9410 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9411 }
9412
9413 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9414 AssertRCBreak(rc);
9415 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9416
9417 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9418 AssertRCBreak(rc);
9419 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9420 }
9421#endif
9422
9423 /*
9424 * PERF_GLOBAL MSR.
9425 */
9426 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9427 {
9428 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9429 AssertRCBreak(rc);
9430 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9431 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9432 }
9433
9434 /*
9435 * PAT MSR.
9436 */
9437 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9438 {
9439 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9440 AssertRCBreak(rc);
9441 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9442 for (unsigned i = 0; i < 8; i++)
9443 {
9444 uint8_t u8Val = (u64Val & 0xff);
9445 if ( u8Val != 0 /* UC */
9446 && u8Val != 1 /* WC */
9447 && u8Val != 4 /* WT */
9448 && u8Val != 5 /* WP */
9449 && u8Val != 6 /* WB */
9450 && u8Val != 7 /* UC- */)
9451 {
9452 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9453 }
9454 u64Val >>= 8;
9455 }
9456 }
9457
9458 /*
9459 * EFER MSR.
9460 */
9461 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9462 {
9463 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9464 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9465 AssertRCBreak(rc);
9466 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9467 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9468 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9469 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9470 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9471 || !(u32GuestCR0 & X86_CR0_PG)
9472 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9473 VMX_IGS_EFER_LMA_LME_MISMATCH);
9474 }
9475
9476 /*
9477 * Segment registers.
9478 */
9479 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9480 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9481 if (!(u32Eflags & X86_EFL_VM))
9482 {
9483 /* CS */
9484 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9485 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9486 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9487 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9488 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9489 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9490 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9491 /* CS cannot be loaded with NULL in protected mode. */
9492 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9493 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9494 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9495 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9496 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9497 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9498 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9499 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9500 else
9501 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9502
9503 /* SS */
9504 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9505 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9506 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9507 if ( !(pCtx->cr0 & X86_CR0_PE)
9508 || pCtx->cs.Attr.n.u4Type == 3)
9509 {
9510 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9511 }
9512 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9513 {
9514 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9515 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9516 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9517 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9518 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9519 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9520 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9521 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9522 }
9523
9524 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9525 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9526 {
9527 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9528 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9529 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9530 || pCtx->ds.Attr.n.u4Type > 11
9531 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9532 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9533 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9534 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9535 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9536 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9537 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9538 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9539 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9540 }
9541 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9542 {
9543 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9544 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9545 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9546 || pCtx->es.Attr.n.u4Type > 11
9547 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9548 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9549 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9550 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9551 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9552 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9553 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9554 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9555 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9556 }
9557 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9558 {
9559 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9560 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9561 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9562 || pCtx->fs.Attr.n.u4Type > 11
9563 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9564 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9565 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9566 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9567 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9568 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9569 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9570 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9571 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9572 }
9573 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9574 {
9575 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9576 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9577 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9578 || pCtx->gs.Attr.n.u4Type > 11
9579 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9580 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9581 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9582 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9583 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9584 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9585 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9586 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9587 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9588 }
9589 /* 64-bit capable CPUs. */
9590#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9591 if (HMVMX_IS_64BIT_HOST_MODE())
9592 {
9593 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9594 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9595 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9596 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9597 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9598 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9599 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9600 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9601 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9602 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9603 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9604 }
9605#endif
9606 }
9607 else
9608 {
9609 /* V86 mode checks. */
9610 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9611 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9612 {
9613 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9614 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9615 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9616 }
9617 else
9618 {
9619 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9620 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9621 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9622 }
9623
9624 /* CS */
9625 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9626 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9627 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9628 /* SS */
9629 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9630 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9631 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9632 /* DS */
9633 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9634 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9635 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9636 /* ES */
9637 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9638 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9639 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9640 /* FS */
9641 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9642 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9643 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9644 /* GS */
9645 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9646 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9647 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9648 /* 64-bit capable CPUs. */
9649#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9650 if (HMVMX_IS_64BIT_HOST_MODE())
9651 {
9652 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9653 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9654 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9655 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9656 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9657 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9658 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9659 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9660 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9661 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9662 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9663 }
9664#endif
9665 }
9666
9667 /*
9668 * TR.
9669 */
9670 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9671 /* 64-bit capable CPUs. */
9672#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9673 if (HMVMX_IS_64BIT_HOST_MODE())
9674 {
9675 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9676 }
9677#endif
9678 if (fLongModeGuest)
9679 {
9680 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9681 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9682 }
9683 else
9684 {
9685 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9686 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9687 VMX_IGS_TR_ATTR_TYPE_INVALID);
9688 }
9689 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9690 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9691 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9692 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9693 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9694 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9695 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9696 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9697
9698 /*
9699 * GDTR and IDTR.
9700 */
9701#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9702 if (HMVMX_IS_64BIT_HOST_MODE())
9703 {
9704 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9705 AssertRCBreak(rc);
9706 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9707
9708 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9709 AssertRCBreak(rc);
9710 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9711 }
9712#endif
9713
9714 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9715 AssertRCBreak(rc);
9716 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9717
9718 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9719 AssertRCBreak(rc);
9720 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9721
9722 /*
9723 * Guest Non-Register State.
9724 */
9725 /* Activity State. */
9726 uint32_t u32ActivityState;
9727 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9728 AssertRCBreak(rc);
9729 HMVMX_CHECK_BREAK( !u32ActivityState
9730 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9731 VMX_IGS_ACTIVITY_STATE_INVALID);
9732 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9733 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9734 uint32_t u32IntrState;
9735 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9736 AssertRCBreak(rc);
9737 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9738 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9739 {
9740 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9741 }
9742
9743 /** @todo Activity state and injecting interrupts. Left as a todo since we
9744 * currently don't use activity states but ACTIVE. */
9745
9746 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9747 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9748
9749 /* Guest interruptibility-state. */
9750 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9751 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9752 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9753 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9754 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9755 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9756 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9757 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9758 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9759 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9760 {
9761 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9762 {
9763 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9764 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9765 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9766 }
9767 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9768 {
9769 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9770 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9771 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9772 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9773 }
9774 }
9775 /** @todo Assumes the processor is not in SMM. */
9776 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9777 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9778 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9779 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9780 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9781 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9782 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9783 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9784 {
9785 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9786 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9787 }
9788
9789 /* Pending debug exceptions. */
9790 if (HMVMX_IS_64BIT_HOST_MODE())
9791 {
9792 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9793 AssertRCBreak(rc);
9794 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9795 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9796 u32Val = u64Val; /* For pending debug exceptions checks below. */
9797 }
9798 else
9799 {
9800 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9801 AssertRCBreak(rc);
9802 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9803 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9804 }
9805
9806 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9807 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9808 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9809 {
9810 if ( (u32Eflags & X86_EFL_TF)
9811 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9812 {
9813 /* Bit 14 is PendingDebug.BS. */
9814 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9815 }
9816 if ( !(u32Eflags & X86_EFL_TF)
9817 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9818 {
9819 /* Bit 14 is PendingDebug.BS. */
9820 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9821 }
9822 }
9823
9824 /* VMCS link pointer. */
9825 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9826 AssertRCBreak(rc);
9827 if (u64Val != UINT64_C(0xffffffffffffffff))
9828 {
9829 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9830 /** @todo Bits beyond the processor's physical-address width MBZ. */
9831 /** @todo 32-bit located in memory referenced by value of this field (as a
9832 * physical address) must contain the processor's VMCS revision ID. */
9833 /** @todo SMM checks. */
9834 }
9835
9836 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9837 * not using Nested Paging? */
9838 if ( pVM->hm.s.fNestedPaging
9839 && !fLongModeGuest
9840 && CPUMIsGuestInPAEModeEx(pCtx))
9841 {
9842 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9843 AssertRCBreak(rc);
9844 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9845
9846 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9847 AssertRCBreak(rc);
9848 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9849
9850 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9851 AssertRCBreak(rc);
9852 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9853
9854 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9855 AssertRCBreak(rc);
9856 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9857 }
9858
9859 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9860 if (uError == VMX_IGS_ERROR)
9861 uError = VMX_IGS_REASON_NOT_FOUND;
9862 } while (0);
9863
9864 pVCpu->hm.s.u32HMError = uError;
9865 return uError;
9866
9867#undef HMVMX_ERROR_BREAK
9868#undef HMVMX_CHECK_BREAK
9869}
9870
9871/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9872/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9873/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9874
9875/** @name VM-exit handlers.
9876 * @{
9877 */
9878
9879/**
9880 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9881 */
9882HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9883{
9884 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9885 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9886 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9887 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9888 return VINF_SUCCESS;
9889 return VINF_EM_RAW_INTERRUPT;
9890}
9891
9892
9893/**
9894 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9895 */
9896HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9897{
9898 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9899 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9900
9901 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9902 AssertRCReturn(rc, rc);
9903
9904 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9905 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9906 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9907 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9908
9909 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9910 {
9911 /*
9912 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9913 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9914 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9915 *
9916 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9917 */
9918 VMXDispatchHostNmi();
9919 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9920 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9921 return VINF_SUCCESS;
9922 }
9923
9924 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9925 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9926 if (RT_UNLIKELY(rc != VINF_SUCCESS))
9927 {
9928 if (rc == VINF_HM_DOUBLE_FAULT)
9929 rc = VINF_SUCCESS;
9930 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9931 return rc;
9932 }
9933
9934 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9935 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9936 switch (uIntType)
9937 {
9938 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9939 Assert(uVector == X86_XCPT_DB);
9940 /* no break */
9941 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9942 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9943 /* no break */
9944 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9945 {
9946 switch (uVector)
9947 {
9948 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9949 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9950 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9951 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9952 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9953 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9954#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9955 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9956 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9957 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9958 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9959 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9960 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9961 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9962 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9963 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9964 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9965 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9966 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9967#endif
9968 default:
9969 {
9970 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9971 AssertRCReturn(rc, rc);
9972
9973 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9974 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9975 {
9976 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9977 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9978 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9979
9980 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9981 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9982 AssertRCReturn(rc, rc);
9983 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9984 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9985 0 /* GCPtrFaultAddress */);
9986 AssertRCReturn(rc, rc);
9987 }
9988 else
9989 {
9990 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9991 pVCpu->hm.s.u32HMError = uVector;
9992 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9993 }
9994 break;
9995 }
9996 }
9997 break;
9998 }
9999
10000 default:
10001 {
10002 pVCpu->hm.s.u32HMError = uExitIntInfo;
10003 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10004 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10005 break;
10006 }
10007 }
10008 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10009 return rc;
10010}
10011
10012
10013/**
10014 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10015 */
10016HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10017{
10018 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10019
10020 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
10021 hmR0VmxClearIntWindowExitVmcs(pVCpu);
10022
10023 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10024 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
10025 return VINF_SUCCESS;
10026}
10027
10028
10029/**
10030 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
10031 */
10032HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10033{
10034 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10035 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
10036 {
10037 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
10038 HMVMX_RETURN_UNEXPECTED_EXIT();
10039 }
10040
10041 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
10042
10043 /*
10044 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
10045 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
10046 */
10047 uint32_t uIntrState = 0;
10048 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10049 AssertRCReturn(rc, rc);
10050
10051 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
10052 if ( fBlockSti
10053 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
10054 {
10055 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10056 }
10057
10058 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
10059 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
10060
10061 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10062 return VINF_SUCCESS;
10063}
10064
10065
10066/**
10067 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10068 */
10069HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10070{
10071 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10072 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10073 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10074}
10075
10076
10077/**
10078 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10079 */
10080HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10081{
10082 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10083 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10084 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10085}
10086
10087
10088/**
10089 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10090 */
10091HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10092{
10093 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10094 PVM pVM = pVCpu->CTX_SUFF(pVM);
10095 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10096 if (RT_LIKELY(rc == VINF_SUCCESS))
10097 {
10098 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10099 Assert(pVmxTransient->cbInstr == 2);
10100 }
10101 else
10102 {
10103 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10104 rc = VERR_EM_INTERPRETER;
10105 }
10106 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10107 return rc;
10108}
10109
10110
10111/**
10112 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10113 */
10114HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10115{
10116 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10117 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10118 AssertRCReturn(rc, rc);
10119
10120 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10121 return VINF_EM_RAW_EMULATE_INSTR;
10122
10123 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10124 HMVMX_RETURN_UNEXPECTED_EXIT();
10125}
10126
10127
10128/**
10129 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10130 */
10131HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10132{
10133 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10134 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10135 AssertRCReturn(rc, rc);
10136
10137 PVM pVM = pVCpu->CTX_SUFF(pVM);
10138 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10139 if (RT_LIKELY(rc == VINF_SUCCESS))
10140 {
10141 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10142 Assert(pVmxTransient->cbInstr == 2);
10143 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10144 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10145 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10146 }
10147 else
10148 rc = VERR_EM_INTERPRETER;
10149 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10150 return rc;
10151}
10152
10153
10154/**
10155 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10156 */
10157HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10158{
10159 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10160 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10161 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10162 AssertRCReturn(rc, rc);
10163
10164 PVM pVM = pVCpu->CTX_SUFF(pVM);
10165 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10166 if (RT_LIKELY(rc == VINF_SUCCESS))
10167 {
10168 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10169 Assert(pVmxTransient->cbInstr == 3);
10170 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10171 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10172 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10173 }
10174 else
10175 {
10176 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10177 rc = VERR_EM_INTERPRETER;
10178 }
10179 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10180 return rc;
10181}
10182
10183
10184/**
10185 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10186 */
10187HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10188{
10189 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10190 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10191 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10192 AssertRCReturn(rc, rc);
10193
10194 PVM pVM = pVCpu->CTX_SUFF(pVM);
10195 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10196 if (RT_LIKELY(rc == VINF_SUCCESS))
10197 {
10198 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10199 Assert(pVmxTransient->cbInstr == 2);
10200 }
10201 else
10202 {
10203 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10204 rc = VERR_EM_INTERPRETER;
10205 }
10206 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10207 return rc;
10208}
10209
10210
10211/**
10212 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10213 */
10214HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10215{
10216 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10217
10218 int rc = VERR_NOT_SUPPORTED;
10219 if (GIMAreHypercallsEnabled(pVCpu))
10220 {
10221 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10222 AssertRCReturn(rc, rc);
10223
10224 rc = GIMHypercall(pVCpu, pMixedCtx);
10225 }
10226 if (rc != VINF_SUCCESS)
10227 {
10228 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10229 rc = VINF_SUCCESS;
10230 }
10231
10232 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10233 return rc;
10234}
10235
10236
10237/**
10238 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10239 */
10240HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10241{
10242 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10243 PVM pVM = pVCpu->CTX_SUFF(pVM);
10244 Assert(!pVM->hm.s.fNestedPaging);
10245
10246 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10247 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10248 AssertRCReturn(rc, rc);
10249
10250 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10251 rc = VBOXSTRICTRC_VAL(rc2);
10252 if (RT_LIKELY(rc == VINF_SUCCESS))
10253 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10254 else
10255 {
10256 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10257 pVmxTransient->uExitQualification, rc));
10258 }
10259 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10260 return rc;
10261}
10262
10263
10264/**
10265 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10266 */
10267HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10268{
10269 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10270 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10271 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10272 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10273 AssertRCReturn(rc, rc);
10274
10275 PVM pVM = pVCpu->CTX_SUFF(pVM);
10276 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10277 if (RT_LIKELY(rc == VINF_SUCCESS))
10278 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10279 else
10280 {
10281 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10282 rc = VERR_EM_INTERPRETER;
10283 }
10284 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10285 return rc;
10286}
10287
10288
10289/**
10290 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10291 */
10292HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10293{
10294 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10295 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10296 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10297 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10298 AssertRCReturn(rc, rc);
10299
10300 PVM pVM = pVCpu->CTX_SUFF(pVM);
10301 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10302 rc = VBOXSTRICTRC_VAL(rc2);
10303 if (RT_LIKELY( rc == VINF_SUCCESS
10304 || rc == VINF_EM_HALT))
10305 {
10306 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10307 AssertRCReturn(rc3, rc3);
10308
10309 if ( rc == VINF_EM_HALT
10310 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10311 {
10312 rc = VINF_SUCCESS;
10313 }
10314 }
10315 else
10316 {
10317 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10318 rc = VERR_EM_INTERPRETER;
10319 }
10320 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10321 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10322 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10323 return rc;
10324}
10325
10326
10327/**
10328 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10329 */
10330HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10331{
10332 /*
10333 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10334 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10335 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10336 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10337 */
10338 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10339 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10340 HMVMX_RETURN_UNEXPECTED_EXIT();
10341}
10342
10343
10344/**
10345 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10346 */
10347HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10348{
10349 /*
10350 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10351 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10352 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10353 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10354 */
10355 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10356 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10357 HMVMX_RETURN_UNEXPECTED_EXIT();
10358}
10359
10360
10361/**
10362 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10363 */
10364HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10365{
10366 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10367 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10368 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10369 HMVMX_RETURN_UNEXPECTED_EXIT();
10370}
10371
10372
10373/**
10374 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10375 */
10376HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10377{
10378 /*
10379 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10380 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10381 * See Intel spec. 25.3 "Other Causes of VM-exits".
10382 */
10383 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10384 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10385 HMVMX_RETURN_UNEXPECTED_EXIT();
10386}
10387
10388
10389/**
10390 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10391 * VM-exit.
10392 */
10393HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10394{
10395 /*
10396 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10397 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10398 *
10399 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10400 * See Intel spec. "23.8 Restrictions on VMX operation".
10401 */
10402 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10403 return VINF_SUCCESS;
10404}
10405
10406
10407/**
10408 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10409 * VM-exit.
10410 */
10411HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10412{
10413 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10414 return VINF_EM_RESET;
10415}
10416
10417
10418/**
10419 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10420 */
10421HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10422{
10423 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10424 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10425 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10426 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10427 AssertRCReturn(rc, rc);
10428
10429 pMixedCtx->rip++;
10430 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10431 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10432 rc = VINF_SUCCESS;
10433 else
10434 rc = VINF_EM_HALT;
10435
10436 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10437 return rc;
10438}
10439
10440
10441/**
10442 * VM-exit handler for instructions that result in a #UD exception delivered to
10443 * the guest.
10444 */
10445HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10446{
10447 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10448 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10449 return VINF_SUCCESS;
10450}
10451
10452
10453/**
10454 * VM-exit handler for expiry of the VMX preemption timer.
10455 */
10456HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10457{
10458 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10459
10460 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10461 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10462
10463 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10464 PVM pVM = pVCpu->CTX_SUFF(pVM);
10465 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10466 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10467 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10468}
10469
10470
10471/**
10472 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10473 */
10474HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10475{
10476 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10477
10478 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10479 /** @todo check if XSETBV is supported by the recompiler. */
10480 return VERR_EM_INTERPRETER;
10481}
10482
10483
10484/**
10485 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10486 */
10487HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10488{
10489 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10490
10491 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10492 /** @todo implement EMInterpretInvpcid() */
10493 return VERR_EM_INTERPRETER;
10494}
10495
10496
10497/**
10498 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10499 * Error VM-exit.
10500 */
10501HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10502{
10503 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10504 AssertRCReturn(rc, rc);
10505
10506 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10507 AssertRCReturn(rc, rc);
10508
10509 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10510 NOREF(uInvalidReason);
10511
10512#ifdef VBOX_STRICT
10513 uint32_t uIntrState;
10514 HMVMXHCUINTREG uHCReg;
10515 uint64_t u64Val;
10516 uint32_t u32Val;
10517
10518 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10519 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10520 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10521 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10522 AssertRCReturn(rc, rc);
10523
10524 Log4(("uInvalidReason %u\n", uInvalidReason));
10525 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10526 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10527 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10528 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10529
10530 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10531 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10532 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10533 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10534 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10535 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10536 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10537 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10538 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10539 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10540 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10541 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10542#else
10543 NOREF(pVmxTransient);
10544#endif
10545
10546 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10547 return VERR_VMX_INVALID_GUEST_STATE;
10548}
10549
10550
10551/**
10552 * VM-exit handler for VM-entry failure due to an MSR-load
10553 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10554 */
10555HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10556{
10557 NOREF(pVmxTransient);
10558 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10559 HMVMX_RETURN_UNEXPECTED_EXIT();
10560}
10561
10562
10563/**
10564 * VM-exit handler for VM-entry failure due to a machine-check event
10565 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10566 */
10567HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10568{
10569 NOREF(pVmxTransient);
10570 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10571 HMVMX_RETURN_UNEXPECTED_EXIT();
10572}
10573
10574
10575/**
10576 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10577 * theory.
10578 */
10579HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10580{
10581 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10582 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10583 return VERR_VMX_UNDEFINED_EXIT_CODE;
10584}
10585
10586
10587/**
10588 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10589 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10590 * Conditional VM-exit.
10591 */
10592HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10593{
10594 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10595
10596 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10597 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10598 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10599 return VERR_EM_INTERPRETER;
10600 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10601 HMVMX_RETURN_UNEXPECTED_EXIT();
10602}
10603
10604
10605/**
10606 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10607 */
10608HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10609{
10610 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10611
10612 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10613 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10614 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10615 return VERR_EM_INTERPRETER;
10616 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10617 HMVMX_RETURN_UNEXPECTED_EXIT();
10618}
10619
10620
10621/**
10622 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10623 */
10624HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10625{
10626 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10627
10628 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10629 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10630 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10631 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10632 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10633 {
10634 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10635 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10636 }
10637 AssertRCReturn(rc, rc);
10638 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10639
10640#ifdef VBOX_STRICT
10641 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10642 {
10643 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10644 && pMixedCtx->ecx != MSR_K6_EFER)
10645 {
10646 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10647 HMVMX_RETURN_UNEXPECTED_EXIT();
10648 }
10649# if HC_ARCH_BITS == 64
10650 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10651 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10652 {
10653 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10654 HMVMX_RETURN_UNEXPECTED_EXIT();
10655 }
10656# endif
10657 }
10658#endif
10659
10660 PVM pVM = pVCpu->CTX_SUFF(pVM);
10661 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10662 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10663 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10664 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10665 if (RT_LIKELY(rc == VINF_SUCCESS))
10666 {
10667 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10668 Assert(pVmxTransient->cbInstr == 2);
10669 }
10670 return rc;
10671}
10672
10673
10674/**
10675 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10676 */
10677HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10678{
10679 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10680 PVM pVM = pVCpu->CTX_SUFF(pVM);
10681 int rc = VINF_SUCCESS;
10682
10683 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10684 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10685 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10686 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10687 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10688 {
10689 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10690 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10691 }
10692 AssertRCReturn(rc, rc);
10693 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10694
10695 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10696 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10697 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10698
10699 if (RT_LIKELY(rc == VINF_SUCCESS))
10700 {
10701 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10702
10703 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10704 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10705 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10706 {
10707 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10708 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10709 EMInterpretWrmsr() changes it. */
10710 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10711 }
10712 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10713 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10714 else if (pMixedCtx->ecx == MSR_K6_EFER)
10715 {
10716 /*
10717 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10718 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10719 * the other bits as well, SCE and NXE. See @bugref{7368}.
10720 */
10721 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10722 }
10723
10724 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10725 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10726 {
10727 switch (pMixedCtx->ecx)
10728 {
10729 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10730 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10731 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10732 case MSR_K8_FS_BASE: /* no break */
10733 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10734 case MSR_K6_EFER: /* already handled above */ break;
10735 default:
10736 {
10737 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10738 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10739#if HC_ARCH_BITS == 64
10740 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10741 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10742#endif
10743 break;
10744 }
10745 }
10746 }
10747#ifdef VBOX_STRICT
10748 else
10749 {
10750 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10751 switch (pMixedCtx->ecx)
10752 {
10753 case MSR_IA32_SYSENTER_CS:
10754 case MSR_IA32_SYSENTER_EIP:
10755 case MSR_IA32_SYSENTER_ESP:
10756 case MSR_K8_FS_BASE:
10757 case MSR_K8_GS_BASE:
10758 {
10759 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10760 HMVMX_RETURN_UNEXPECTED_EXIT();
10761 }
10762
10763 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10764 default:
10765 {
10766 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10767 {
10768 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10769 if (pMixedCtx->ecx != MSR_K6_EFER)
10770 {
10771 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10772 pMixedCtx->ecx));
10773 HMVMX_RETURN_UNEXPECTED_EXIT();
10774 }
10775 }
10776
10777#if HC_ARCH_BITS == 64
10778 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10779 {
10780 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10781 HMVMX_RETURN_UNEXPECTED_EXIT();
10782 }
10783#endif
10784 break;
10785 }
10786 }
10787 }
10788#endif /* VBOX_STRICT */
10789 }
10790 return rc;
10791}
10792
10793
10794/**
10795 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10796 */
10797HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10798{
10799 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10800
10801 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10802 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10803 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10804 return VERR_EM_INTERPRETER;
10805 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10806 HMVMX_RETURN_UNEXPECTED_EXIT();
10807}
10808
10809
10810/**
10811 * VM-exit handler for when the TPR value is lowered below the specified
10812 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10813 */
10814HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10815{
10816 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10817 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10818
10819 /*
10820 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10821 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10822 * resume guest execution.
10823 */
10824 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10825 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10826 return VINF_SUCCESS;
10827}
10828
10829
10830/**
10831 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10832 * VM-exit.
10833 *
10834 * @retval VINF_SUCCESS when guest execution can continue.
10835 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10836 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10837 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10838 * recompiler.
10839 */
10840HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10841{
10842 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10843 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10844 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10845 AssertRCReturn(rc, rc);
10846
10847 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10848 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10849 PVM pVM = pVCpu->CTX_SUFF(pVM);
10850 switch (uAccessType)
10851 {
10852 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10853 {
10854#if 0
10855 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10856 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10857#else
10858 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10859 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10860 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10861#endif
10862 AssertRCReturn(rc, rc);
10863
10864 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10865 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10866 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10867 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10868
10869 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10870 {
10871 case 0: /* CR0 */
10872 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10873 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10874 break;
10875 case 2: /* CR2 */
10876 /* Nothing to do here, CR2 it's not part of the VMCS. */
10877 break;
10878 case 3: /* CR3 */
10879 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10880 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10881 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10882 break;
10883 case 4: /* CR4 */
10884 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10885 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10886 break;
10887 case 8: /* CR8 */
10888 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10889 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10890 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10891 break;
10892 default:
10893 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10894 break;
10895 }
10896
10897 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10898 break;
10899 }
10900
10901 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10902 {
10903 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10904 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10905 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10906 AssertRCReturn(rc, rc);
10907 Assert( !pVM->hm.s.fNestedPaging
10908 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10909 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10910
10911 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10912 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10913 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10914
10915 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10916 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10917 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10918 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10919 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10920 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10921 break;
10922 }
10923
10924 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10925 {
10926 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10927 AssertRCReturn(rc, rc);
10928 rc = EMInterpretCLTS(pVM, pVCpu);
10929 AssertRCReturn(rc, rc);
10930 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10931 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10932 Log4(("CRX CLTS write rc=%d\n", rc));
10933 break;
10934 }
10935
10936 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10937 {
10938 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10939 AssertRCReturn(rc, rc);
10940 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10941 if (RT_LIKELY(rc == VINF_SUCCESS))
10942 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10943 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10944 Log4(("CRX LMSW write rc=%d\n", rc));
10945 break;
10946 }
10947
10948 default:
10949 {
10950 AssertMsgFailed(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType));
10951 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10952 }
10953 }
10954
10955 /* Validate possible error codes. */
10956 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10957 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10958 if (RT_SUCCESS(rc))
10959 {
10960 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10961 AssertRCReturn(rc2, rc2);
10962 }
10963
10964 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10965 return rc;
10966}
10967
10968
10969/**
10970 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10971 * VM-exit.
10972 */
10973HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10974{
10975 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10976 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10977
10978 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10979 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10980 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10981 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10982 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10983 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10984 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10985 AssertRCReturn(rc2, rc2);
10986
10987 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10988 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10989 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10990 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10991 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10992 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10993 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10994 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10995
10996 /* I/O operation lookup arrays. */
10997 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10998 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10999
11000 VBOXSTRICTRC rcStrict;
11001 uint32_t const cbValue = s_aIOSizes[uIOWidth];
11002 uint32_t const cbInstr = pVmxTransient->cbInstr;
11003 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
11004 PVM pVM = pVCpu->CTX_SUFF(pVM);
11005 if (fIOString)
11006 {
11007#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
11008 /*
11009 * INS/OUTS - I/O String instruction.
11010 *
11011 * Use instruction-information if available, otherwise fall back on
11012 * interpreting the instruction.
11013 */
11014 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11015 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
11016 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
11017 {
11018 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11019 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11020 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11021 AssertRCReturn(rc2, rc2);
11022 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
11023 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
11024 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
11025 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
11026 if (fIOWrite)
11027 {
11028 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
11029 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
11030 }
11031 else
11032 {
11033 /*
11034 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
11035 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
11036 * See Intel Instruction spec. for "INS".
11037 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
11038 */
11039 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
11040 }
11041 }
11042 else
11043 {
11044 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11045 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11046 AssertRCReturn(rc2, rc2);
11047 rcStrict = IEMExecOne(pVCpu);
11048 }
11049 /** @todo IEM needs to be setting these flags somehow. */
11050 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11051 fUpdateRipAlready = true;
11052#else
11053 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11054 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
11055 if (RT_SUCCESS(rcStrict))
11056 {
11057 if (fIOWrite)
11058 {
11059 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11060 (DISCPUMODE)pDis->uAddrMode, cbValue);
11061 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
11062 }
11063 else
11064 {
11065 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11066 (DISCPUMODE)pDis->uAddrMode, cbValue);
11067 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11068 }
11069 }
11070 else
11071 {
11072 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
11073 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11074 }
11075#endif
11076 }
11077 else
11078 {
11079 /*
11080 * IN/OUT - I/O instruction.
11081 */
11082 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11083 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11084 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11085 if (fIOWrite)
11086 {
11087 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11088 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11089 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11090 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11091 }
11092 else
11093 {
11094 uint32_t u32Result = 0;
11095 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11096 if (IOM_SUCCESS(rcStrict))
11097 {
11098 /* Save result of I/O IN instr. in AL/AX/EAX. */
11099 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11100 }
11101 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11102 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11103 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11104 }
11105 }
11106
11107 if (IOM_SUCCESS(rcStrict))
11108 {
11109 if (!fUpdateRipAlready)
11110 {
11111 pMixedCtx->rip += cbInstr;
11112 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11113 }
11114
11115 /*
11116 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11117 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11118 */
11119 if (fIOString)
11120 {
11121 /** @todo Single-step for INS/OUTS with REP prefix? */
11122 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11123 }
11124 else if (fStepping)
11125 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11126
11127 /*
11128 * If any I/O breakpoints are armed, we need to check if one triggered
11129 * and take appropriate action.
11130 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11131 */
11132 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11133 AssertRCReturn(rc2, rc2);
11134
11135 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11136 * execution engines about whether hyper BPs and such are pending. */
11137 uint32_t const uDr7 = pMixedCtx->dr[7];
11138 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11139 && X86_DR7_ANY_RW_IO(uDr7)
11140 && (pMixedCtx->cr4 & X86_CR4_DE))
11141 || DBGFBpIsHwIoArmed(pVM)))
11142 {
11143 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11144
11145 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11146 VMMRZCallRing3Disable(pVCpu);
11147 HM_DISABLE_PREEMPT_IF_NEEDED();
11148
11149 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11150
11151 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11152 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11153 {
11154 /* Raise #DB. */
11155 if (fIsGuestDbgActive)
11156 ASMSetDR6(pMixedCtx->dr[6]);
11157 if (pMixedCtx->dr[7] != uDr7)
11158 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11159
11160 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11161 }
11162 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11163 else if ( rcStrict2 != VINF_SUCCESS
11164 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11165 rcStrict = rcStrict2;
11166
11167 HM_RESTORE_PREEMPT_IF_NEEDED();
11168 VMMRZCallRing3Enable(pVCpu);
11169 }
11170 }
11171
11172#ifdef DEBUG
11173 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11174 Assert(!fIOWrite);
11175 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11176 Assert(fIOWrite);
11177 else
11178 {
11179 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11180 * statuses, that the VMM device and some others may return. See
11181 * IOM_SUCCESS() for guidance. */
11182 AssertMsg( RT_FAILURE(rcStrict)
11183 || rcStrict == VINF_SUCCESS
11184 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11185 || rcStrict == VINF_EM_DBG_BREAKPOINT
11186 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11187 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11188 }
11189#endif
11190
11191 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11192 return VBOXSTRICTRC_TODO(rcStrict);
11193}
11194
11195
11196/**
11197 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11198 * VM-exit.
11199 */
11200HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11201{
11202 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11203
11204 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11205 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11206 AssertRCReturn(rc, rc);
11207 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11208 {
11209 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11210 AssertRCReturn(rc, rc);
11211 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11212 {
11213 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11214
11215 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11216 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11217
11218 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11219 Assert(!pVCpu->hm.s.Event.fPending);
11220 pVCpu->hm.s.Event.fPending = true;
11221 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11222 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11223 AssertRCReturn(rc, rc);
11224 if (fErrorCodeValid)
11225 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11226 else
11227 pVCpu->hm.s.Event.u32ErrCode = 0;
11228 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11229 && uVector == X86_XCPT_PF)
11230 {
11231 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11232 }
11233
11234 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11235 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11236 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11237 }
11238 }
11239
11240 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11241 * emulation. */
11242 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11243 return VERR_EM_INTERPRETER;
11244}
11245
11246
11247/**
11248 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11249 */
11250HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11251{
11252 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11253 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11254 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11255 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11256 AssertRCReturn(rc, rc);
11257 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11258 return VINF_EM_DBG_STEPPED;
11259}
11260
11261
11262/**
11263 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11264 */
11265HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11266{
11267 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11268
11269 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11270 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11271 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11272 {
11273 if (rc == VINF_HM_DOUBLE_FAULT)
11274 rc = VINF_SUCCESS;
11275 return rc;
11276 }
11277
11278#if 0
11279 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11280 * just sync the whole thing. */
11281 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11282#else
11283 /* Aggressive state sync. for now. */
11284 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11285 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11286 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11287#endif
11288 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11289 AssertRCReturn(rc, rc);
11290
11291 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11292 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11293 switch (uAccessType)
11294 {
11295 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11296 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11297 {
11298 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11299 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11300 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11301
11302 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11303 GCPhys &= PAGE_BASE_GC_MASK;
11304 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11305 PVM pVM = pVCpu->CTX_SUFF(pVM);
11306 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11307 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11308
11309 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11310 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11311 CPUMCTX2CORE(pMixedCtx), GCPhys);
11312 rc = VBOXSTRICTRC_VAL(rc2);
11313 Log4(("ApicAccess rc=%d\n", rc));
11314 if ( rc == VINF_SUCCESS
11315 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11316 || rc == VERR_PAGE_NOT_PRESENT)
11317 {
11318 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11319 | HM_CHANGED_GUEST_RSP
11320 | HM_CHANGED_GUEST_RFLAGS
11321 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11322 rc = VINF_SUCCESS;
11323 }
11324 break;
11325 }
11326
11327 default:
11328 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11329 rc = VINF_EM_RAW_EMULATE_INSTR;
11330 break;
11331 }
11332
11333 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11334 if (rc != VINF_SUCCESS)
11335 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccessToR3);
11336 return rc;
11337}
11338
11339
11340/**
11341 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11342 * VM-exit.
11343 */
11344HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11345{
11346 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11347
11348 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11349 if (pVmxTransient->fWasGuestDebugStateActive)
11350 {
11351 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11352 HMVMX_RETURN_UNEXPECTED_EXIT();
11353 }
11354
11355 int rc = VERR_INTERNAL_ERROR_5;
11356 if ( !DBGFIsStepping(pVCpu)
11357 && !pVCpu->hm.s.fSingleInstruction
11358 && !pVmxTransient->fWasHyperDebugStateActive)
11359 {
11360 /* Don't intercept MOV DRx and #DB any more. */
11361 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11362 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11363 AssertRCReturn(rc, rc);
11364
11365 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11366 {
11367#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11368 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11369 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11370 AssertRCReturn(rc, rc);
11371#endif
11372 }
11373
11374 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11375 VMMRZCallRing3Disable(pVCpu);
11376 HM_DISABLE_PREEMPT_IF_NEEDED();
11377
11378 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11379 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11380 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11381
11382 HM_RESTORE_PREEMPT_IF_NEEDED();
11383 VMMRZCallRing3Enable(pVCpu);
11384
11385#ifdef VBOX_WITH_STATISTICS
11386 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11387 AssertRCReturn(rc, rc);
11388 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11389 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11390 else
11391 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11392#endif
11393 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11394 return VINF_SUCCESS;
11395 }
11396
11397 /*
11398 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11399 * Update the segment registers and DR7 from the CPU.
11400 */
11401 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11402 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11403 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11404 AssertRCReturn(rc, rc);
11405 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11406
11407 PVM pVM = pVCpu->CTX_SUFF(pVM);
11408 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11409 {
11410 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11411 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11412 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11413 if (RT_SUCCESS(rc))
11414 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11415 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11416 }
11417 else
11418 {
11419 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11420 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11421 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11422 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11423 }
11424
11425 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11426 if (RT_SUCCESS(rc))
11427 {
11428 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11429 AssertRCReturn(rc2, rc2);
11430 }
11431 return rc;
11432}
11433
11434
11435/**
11436 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11437 * Conditional VM-exit.
11438 */
11439HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11440{
11441 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11442 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11443
11444 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11445 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11446 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11447 {
11448 if (rc == VINF_HM_DOUBLE_FAULT)
11449 rc = VINF_SUCCESS;
11450 return rc;
11451 }
11452
11453 RTGCPHYS GCPhys = 0;
11454 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11455
11456#if 0
11457 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11458#else
11459 /* Aggressive state sync. for now. */
11460 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11461 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11462 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11463#endif
11464 AssertRCReturn(rc, rc);
11465
11466 /*
11467 * If we succeed, resume guest execution.
11468 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11469 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11470 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11471 * weird case. See @bugref{6043}.
11472 */
11473 PVM pVM = pVCpu->CTX_SUFF(pVM);
11474 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11475 rc = VBOXSTRICTRC_VAL(rc2);
11476 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11477 if ( rc == VINF_SUCCESS
11478 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11479 || rc == VERR_PAGE_NOT_PRESENT)
11480 {
11481 /* Successfully handled MMIO operation. */
11482 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11483 | HM_CHANGED_GUEST_RSP
11484 | HM_CHANGED_GUEST_RFLAGS
11485 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11486 rc = VINF_SUCCESS;
11487 }
11488 return rc;
11489}
11490
11491
11492/**
11493 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11494 * VM-exit.
11495 */
11496HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11497{
11498 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11499 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11500
11501 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11502 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11503 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11504 {
11505 if (rc == VINF_HM_DOUBLE_FAULT)
11506 rc = VINF_SUCCESS;
11507 return rc;
11508 }
11509
11510 RTGCPHYS GCPhys = 0;
11511 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11512 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11513#if 0
11514 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11515#else
11516 /* Aggressive state sync. for now. */
11517 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11518 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11519 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11520#endif
11521 AssertRCReturn(rc, rc);
11522
11523 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11524 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11525
11526 RTGCUINT uErrorCode = 0;
11527 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11528 uErrorCode |= X86_TRAP_PF_ID;
11529 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11530 uErrorCode |= X86_TRAP_PF_RW;
11531 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11532 uErrorCode |= X86_TRAP_PF_P;
11533
11534 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11535
11536 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11537 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11538
11539 /* Handle the pagefault trap for the nested shadow table. */
11540 PVM pVM = pVCpu->CTX_SUFF(pVM);
11541 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11542 TRPMResetTrap(pVCpu);
11543
11544 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11545 if ( rc == VINF_SUCCESS
11546 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11547 || rc == VERR_PAGE_NOT_PRESENT)
11548 {
11549 /* Successfully synced our nested page tables. */
11550 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11551 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11552 | HM_CHANGED_GUEST_RSP
11553 | HM_CHANGED_GUEST_RFLAGS);
11554 return VINF_SUCCESS;
11555 }
11556
11557 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11558 return rc;
11559}
11560
11561/** @} */
11562
11563/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11564/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11565/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11566
11567/** @name VM-exit exception handlers.
11568 * @{
11569 */
11570
11571/**
11572 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11573 */
11574static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11575{
11576 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11577 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11578
11579 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11580 AssertRCReturn(rc, rc);
11581
11582 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11583 {
11584 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11585 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11586
11587 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11588 * provides VM-exit instruction length. If this causes problem later,
11589 * disassemble the instruction like it's done on AMD-V. */
11590 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11591 AssertRCReturn(rc2, rc2);
11592 return rc;
11593 }
11594
11595 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11596 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11597 return rc;
11598}
11599
11600
11601/**
11602 * VM-exit exception handler for #BP (Breakpoint exception).
11603 */
11604static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11605{
11606 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11607 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11608
11609 /** @todo Try optimize this by not saving the entire guest state unless
11610 * really needed. */
11611 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11612 AssertRCReturn(rc, rc);
11613
11614 PVM pVM = pVCpu->CTX_SUFF(pVM);
11615 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11616 if (rc == VINF_EM_RAW_GUEST_TRAP)
11617 {
11618 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11619 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11620 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11621 AssertRCReturn(rc, rc);
11622
11623 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11624 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11625 }
11626
11627 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11628 return rc;
11629}
11630
11631
11632/**
11633 * VM-exit exception handler for #DB (Debug exception).
11634 */
11635static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11636{
11637 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11638 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11639 Log6(("XcptDB\n"));
11640
11641 /*
11642 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11643 * for processing.
11644 */
11645 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11646 AssertRCReturn(rc, rc);
11647
11648 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11649 uint64_t uDR6 = X86_DR6_INIT_VAL;
11650 uDR6 |= ( pVmxTransient->uExitQualification
11651 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11652
11653 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11654 if (rc == VINF_EM_RAW_GUEST_TRAP)
11655 {
11656 /*
11657 * The exception was for the guest. Update DR6, DR7.GD and
11658 * IA32_DEBUGCTL.LBR before forwarding it.
11659 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11660 */
11661 VMMRZCallRing3Disable(pVCpu);
11662 HM_DISABLE_PREEMPT_IF_NEEDED();
11663
11664 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11665 pMixedCtx->dr[6] |= uDR6;
11666 if (CPUMIsGuestDebugStateActive(pVCpu))
11667 ASMSetDR6(pMixedCtx->dr[6]);
11668
11669 HM_RESTORE_PREEMPT_IF_NEEDED();
11670 VMMRZCallRing3Enable(pVCpu);
11671
11672 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11673 AssertRCReturn(rc, rc);
11674
11675 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11676 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11677
11678 /* Paranoia. */
11679 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11680 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11681
11682 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11683 AssertRCReturn(rc, rc);
11684
11685 /*
11686 * Raise #DB in the guest.
11687 *
11688 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11689 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11690 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11691 *
11692 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11693 */
11694 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11695 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11696 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11697 AssertRCReturn(rc, rc);
11698 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11699 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11700 return VINF_SUCCESS;
11701 }
11702
11703 /*
11704 * Not a guest trap, must be a hypervisor related debug event then.
11705 * Update DR6 in case someone is interested in it.
11706 */
11707 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11708 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11709 CPUMSetHyperDR6(pVCpu, uDR6);
11710
11711 return rc;
11712}
11713
11714
11715/**
11716 * VM-exit exception handler for #NM (Device-not-available exception: floating
11717 * point exception).
11718 */
11719static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11720{
11721 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11722
11723 /* We require CR0 and EFER. EFER is always up-to-date. */
11724 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11725 AssertRCReturn(rc, rc);
11726
11727 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11728 VMMRZCallRing3Disable(pVCpu);
11729 HM_DISABLE_PREEMPT_IF_NEEDED();
11730
11731 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11732 if (pVmxTransient->fWasGuestFPUStateActive)
11733 {
11734 rc = VINF_EM_RAW_GUEST_TRAP;
11735 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11736 }
11737 else
11738 {
11739#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11740 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11741#endif
11742 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11743 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11744 }
11745
11746 HM_RESTORE_PREEMPT_IF_NEEDED();
11747 VMMRZCallRing3Enable(pVCpu);
11748
11749 if (rc == VINF_SUCCESS)
11750 {
11751 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11752 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11753 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11754 pVCpu->hm.s.fUseGuestFpu = true;
11755 }
11756 else
11757 {
11758 /* Forward #NM to the guest. */
11759 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11760 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11761 AssertRCReturn(rc, rc);
11762 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11763 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11764 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11765 }
11766
11767 return VINF_SUCCESS;
11768}
11769
11770
11771/**
11772 * VM-exit exception handler for #GP (General-protection exception).
11773 *
11774 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11775 */
11776static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11777{
11778 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11779 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11780
11781 int rc = VERR_INTERNAL_ERROR_5;
11782 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11783 {
11784#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11785 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11786 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11787 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11788 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11789 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11790 AssertRCReturn(rc, rc);
11791 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11792 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11793 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11794 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11795 return rc;
11796#else
11797 /* We don't intercept #GP. */
11798 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11799 NOREF(pVmxTransient);
11800 return VERR_VMX_UNEXPECTED_EXCEPTION;
11801#endif
11802 }
11803
11804 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11805 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11806
11807 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11808 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11809 AssertRCReturn(rc, rc);
11810
11811 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11812 uint32_t cbOp = 0;
11813 PVM pVM = pVCpu->CTX_SUFF(pVM);
11814 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11815 if (RT_SUCCESS(rc))
11816 {
11817 rc = VINF_SUCCESS;
11818 Assert(cbOp == pDis->cbInstr);
11819 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11820 switch (pDis->pCurInstr->uOpcode)
11821 {
11822 case OP_CLI:
11823 {
11824 pMixedCtx->eflags.Bits.u1IF = 0;
11825 pMixedCtx->eflags.Bits.u1RF = 0;
11826 pMixedCtx->rip += pDis->cbInstr;
11827 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11828 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11829 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11830 break;
11831 }
11832
11833 case OP_STI:
11834 {
11835 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11836 pMixedCtx->eflags.Bits.u1IF = 1;
11837 pMixedCtx->eflags.Bits.u1RF = 0;
11838 pMixedCtx->rip += pDis->cbInstr;
11839 if (!fOldIF)
11840 {
11841 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11842 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11843 }
11844 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11845 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11846 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11847 break;
11848 }
11849
11850 case OP_HLT:
11851 {
11852 rc = VINF_EM_HALT;
11853 pMixedCtx->rip += pDis->cbInstr;
11854 pMixedCtx->eflags.Bits.u1RF = 0;
11855 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11857 break;
11858 }
11859
11860 case OP_POPF:
11861 {
11862 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11863 uint32_t cbParm;
11864 uint32_t uMask;
11865 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11866 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11867 {
11868 cbParm = 4;
11869 uMask = 0xffffffff;
11870 }
11871 else
11872 {
11873 cbParm = 2;
11874 uMask = 0xffff;
11875 }
11876
11877 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11878 RTGCPTR GCPtrStack = 0;
11879 X86EFLAGS Eflags;
11880 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11881 &GCPtrStack);
11882 if (RT_SUCCESS(rc))
11883 {
11884 Assert(sizeof(Eflags.u32) >= cbParm);
11885 Eflags.u32 = 0;
11886 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11887 }
11888 if (RT_FAILURE(rc))
11889 {
11890 rc = VERR_EM_INTERPRETER;
11891 break;
11892 }
11893 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11894 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
11895 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11896 pMixedCtx->esp += cbParm;
11897 pMixedCtx->esp &= uMask;
11898 pMixedCtx->rip += pDis->cbInstr;
11899 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11900 | HM_CHANGED_GUEST_RSP
11901 | HM_CHANGED_GUEST_RFLAGS);
11902 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11903 if (fStepping)
11904 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11905
11906 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11907 break;
11908 }
11909
11910 case OP_PUSHF:
11911 {
11912 uint32_t cbParm;
11913 uint32_t uMask;
11914 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11915 {
11916 cbParm = 4;
11917 uMask = 0xffffffff;
11918 }
11919 else
11920 {
11921 cbParm = 2;
11922 uMask = 0xffff;
11923 }
11924
11925 /* Get the stack pointer & push the contents of eflags onto the stack. */
11926 RTGCPTR GCPtrStack = 0;
11927 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11928 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11929 if (RT_FAILURE(rc))
11930 {
11931 rc = VERR_EM_INTERPRETER;
11932 break;
11933 }
11934 X86EFLAGS Eflags = pMixedCtx->eflags;
11935 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11936 Eflags.Bits.u1RF = 0;
11937 Eflags.Bits.u1VM = 0;
11938
11939 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11940 if (RT_FAILURE(rc))
11941 {
11942 rc = VERR_EM_INTERPRETER;
11943 break;
11944 }
11945 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11946 pMixedCtx->esp -= cbParm;
11947 pMixedCtx->esp &= uMask;
11948 pMixedCtx->rip += pDis->cbInstr;
11949 pMixedCtx->eflags.Bits.u1RF = 0;
11950 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11951 | HM_CHANGED_GUEST_RSP
11952 | HM_CHANGED_GUEST_RFLAGS);
11953 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11954 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11955 break;
11956 }
11957
11958 case OP_IRET:
11959 {
11960 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11961 * instruction reference. */
11962 RTGCPTR GCPtrStack = 0;
11963 uint32_t uMask = 0xffff;
11964 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11965 uint16_t aIretFrame[3];
11966 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11967 {
11968 rc = VERR_EM_INTERPRETER;
11969 break;
11970 }
11971 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11972 &GCPtrStack);
11973 if (RT_SUCCESS(rc))
11974 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11975 if (RT_FAILURE(rc))
11976 {
11977 rc = VERR_EM_INTERPRETER;
11978 break;
11979 }
11980 pMixedCtx->eip = 0;
11981 pMixedCtx->ip = aIretFrame[0];
11982 pMixedCtx->cs.Sel = aIretFrame[1];
11983 pMixedCtx->cs.ValidSel = aIretFrame[1];
11984 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11985 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
11986 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11987 pMixedCtx->sp += sizeof(aIretFrame);
11988 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11989 | HM_CHANGED_GUEST_SEGMENT_REGS
11990 | HM_CHANGED_GUEST_RSP
11991 | HM_CHANGED_GUEST_RFLAGS);
11992 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11993 if (fStepping)
11994 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11995 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11996 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11997 break;
11998 }
11999
12000 case OP_INT:
12001 {
12002 uint16_t uVector = pDis->Param1.uValue & 0xff;
12003 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
12004 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12005 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12006 break;
12007 }
12008
12009 case OP_INTO:
12010 {
12011 if (pMixedCtx->eflags.Bits.u1OF)
12012 {
12013 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
12014 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12015 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12016 }
12017 else
12018 {
12019 pMixedCtx->eflags.Bits.u1RF = 0;
12020 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12021 }
12022 break;
12023 }
12024
12025 default:
12026 {
12027 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
12028 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
12029 EMCODETYPE_SUPERVISOR);
12030 rc = VBOXSTRICTRC_VAL(rc2);
12031 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
12032 /** @todo We have to set pending-debug exceptions here when the guest is
12033 * single-stepping depending on the instruction that was interpreted. */
12034 Log4(("#GP rc=%Rrc\n", rc));
12035 break;
12036 }
12037 }
12038 }
12039 else
12040 rc = VERR_EM_INTERPRETER;
12041
12042 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
12043 ("#GP Unexpected rc=%Rrc\n", rc));
12044 return rc;
12045}
12046
12047
12048#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12049/**
12050 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
12051 * the exception reported in the VMX transient structure back into the VM.
12052 *
12053 * @remarks Requires uExitIntInfo in the VMX transient structure to be
12054 * up-to-date.
12055 */
12056static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12057{
12058 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12059
12060 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12061 hmR0VmxCheckExitDueToEventDelivery(). */
12062 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12063 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12064 AssertRCReturn(rc, rc);
12065 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12066
12067#ifdef DEBUG_ramshankar
12068 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12069 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12070 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12071#endif
12072
12073 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12074 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12075 return VINF_SUCCESS;
12076}
12077#endif
12078
12079
12080/**
12081 * VM-exit exception handler for #PF (Page-fault exception).
12082 */
12083static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12084{
12085 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12086 PVM pVM = pVCpu->CTX_SUFF(pVM);
12087 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12088 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12089 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12090 AssertRCReturn(rc, rc);
12091
12092#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12093 if (pVM->hm.s.fNestedPaging)
12094 {
12095 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12096 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12097 {
12098 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12099 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12100 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12101 }
12102 else
12103 {
12104 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12105 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12106 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12107 }
12108 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12109 return rc;
12110 }
12111#else
12112 Assert(!pVM->hm.s.fNestedPaging);
12113 NOREF(pVM);
12114#endif
12115
12116 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12117 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12118 if (pVmxTransient->fVectoringPF)
12119 {
12120 Assert(pVCpu->hm.s.Event.fPending);
12121 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12122 }
12123
12124 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12125 AssertRCReturn(rc, rc);
12126
12127 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12128 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12129
12130 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12131 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12132 (RTGCPTR)pVmxTransient->uExitQualification);
12133
12134 Log4(("#PF: rc=%Rrc\n", rc));
12135 if (rc == VINF_SUCCESS)
12136 {
12137 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12138 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12139 * memory? We don't update the whole state here... */
12140 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12141 | HM_CHANGED_GUEST_RSP
12142 | HM_CHANGED_GUEST_RFLAGS
12143 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12144 TRPMResetTrap(pVCpu);
12145 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12146 return rc;
12147 }
12148
12149 if (rc == VINF_EM_RAW_GUEST_TRAP)
12150 {
12151 if (!pVmxTransient->fVectoringDoublePF)
12152 {
12153 /* It's a guest page fault and needs to be reflected to the guest. */
12154 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12155 TRPMResetTrap(pVCpu);
12156 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12157 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12158 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12159 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12160 }
12161 else
12162 {
12163 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12164 TRPMResetTrap(pVCpu);
12165 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12166 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12167 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12168 }
12169
12170 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12171 return VINF_SUCCESS;
12172 }
12173
12174 TRPMResetTrap(pVCpu);
12175 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12176 return rc;
12177}
12178
12179/** @} */
12180
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette