VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 48649

Last change on this file since 48649 was 48649, checked in by vboxsync, 12 years ago

VMM/HMVMXR0: VMCS size is bits 44:32 not 43:32. Assert what the Intel spec. claims as well.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 453.1 KB
Line 
1/* $Id: HMVMXR0.cpp 48649 2013-09-24 09:05:56Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#define HMVMX_ALWAYS_SWAP_FPU_STATE
45#endif
46
47
48/*******************************************************************************
49* Defined Constants And Macros *
50*******************************************************************************/
51#if defined(RT_ARCH_AMD64)
52# define HMVMX_IS_64BIT_HOST_MODE() (true)
53typedef RTHCUINTREG HMVMXHCUINTREG;
54#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
55extern "C" uint32_t g_fVMXIs64bitHost;
56# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
57typedef uint64_t HMVMXHCUINTREG;
58#else
59# define HMVMX_IS_64BIT_HOST_MODE() (false)
60typedef RTHCUINTREG HMVMXHCUINTREG;
61#endif
62
63/** Use the function table. */
64#define HMVMX_USE_FUNCTION_TABLE
65
66/** Determine which tagged-TLB flush handler to use. */
67#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72/** @name Updated-guest-state flags.
73 * @{ */
74#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95 | HMVMX_UPDATED_GUEST_RSP \
96 | HMVMX_UPDATED_GUEST_RFLAGS \
97 | HMVMX_UPDATED_GUEST_CR0 \
98 | HMVMX_UPDATED_GUEST_CR3 \
99 | HMVMX_UPDATED_GUEST_CR4 \
100 | HMVMX_UPDATED_GUEST_GDTR \
101 | HMVMX_UPDATED_GUEST_IDTR \
102 | HMVMX_UPDATED_GUEST_LDTR \
103 | HMVMX_UPDATED_GUEST_TR \
104 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105 | HMVMX_UPDATED_GUEST_DEBUG \
106 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113 | HMVMX_UPDATED_GUEST_APIC_STATE)
114/** @} */
115
116/** @name
117 * Flags to skip redundant reads of some common VMCS fields that are not part of
118 * the guest-CPU state but are in the transient structure.
119 */
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126/** @} */
127
128/** @name
129 * States of the VMCS.
130 *
131 * This does not reflect all possible VMCS states but currently only those
132 * needed for maintaining the VMCS consistently even when thread-context hooks
133 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
134 */
135#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
136#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
137#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
138/** @} */
139
140/**
141 * Exception bitmap mask for real-mode guests (real-on-v86).
142 *
143 * We need to intercept all exceptions manually (except #PF). #NM is also
144 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
145 * even in real-mode if we have Nested Paging support.
146 */
147#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
148 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
149 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
150 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
151 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
153 | RT_BIT(X86_XCPT_XF))
154
155/**
156 * Exception bitmap mask for all contributory exceptions.
157 *
158 * Page fault is deliberately excluded here as it's conditional as to whether
159 * it's contributory or benign. Page faults are handled separately.
160 */
161#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
162 | RT_BIT(X86_XCPT_DE))
163
164/** Maximum VM-instruction error number. */
165#define HMVMX_INSTR_ERROR_MAX 28
166
167/** Profiling macro. */
168#ifdef HM_PROFILE_EXIT_DISPATCH
169# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171#else
172# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174#endif
175
176/** Assert that preemption is disabled or covered by thread-context hooks. */
177#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
178 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180/** Assert that we haven't migrated CPUs when thread-context hooks are not
181 * used. */
182#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184 ("Illegal migration! Entered on CPU %u Current %u\n", \
185 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187/*******************************************************************************
188* Structures and Typedefs *
189*******************************************************************************/
190/**
191 * VMX transient state.
192 *
193 * A state structure for holding miscellaneous information across
194 * VMX non-root operation and restored after the transition.
195 */
196typedef struct VMXTRANSIENT
197{
198 /** The host's rflags/eflags. */
199 RTCCUINTREG uEflags;
200#if HC_ARCH_BITS == 32
201 uint32_t u32Alignment0;
202#endif
203 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
204 uint64_t u64LStarMsr;
205 /** The guest's TPR value used for TPR shadowing. */
206 uint8_t u8GuestTpr;
207 /** Alignment. */
208 uint8_t abAlignment0[7];
209
210 /** The basic VM-exit reason. */
211 uint16_t uExitReason;
212 /** Alignment. */
213 uint16_t u16Alignment0;
214 /** The VM-exit interruption error code. */
215 uint32_t uExitIntrErrorCode;
216 /** The VM-exit exit qualification. */
217 uint64_t uExitQualification;
218
219 /** The VM-exit interruption-information field. */
220 uint32_t uExitIntrInfo;
221 /** The VM-exit instruction-length field. */
222 uint32_t cbInstr;
223 /** The VM-exit instruction-information field. */
224 union
225 {
226 /** Plain unsigned int representation. */
227 uint32_t u;
228 /** INS and OUTS information. */
229 struct
230 {
231 uint32_t u6Reserved0 : 6;
232 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
233 uint32_t u3AddrSize : 3;
234 uint32_t u5Reserved1 : 5;
235 /** The segment register (X86_SREG_XXX). */
236 uint32_t iSegReg : 3;
237 uint32_t uReserved2 : 14;
238 } StrIo;
239 } ExitInstrInfo;
240 /** Whether the VM-entry failed or not. */
241 bool fVMEntryFailed;
242 /** Alignment. */
243 uint8_t abAlignment1[3];
244
245 /** The VM-entry interruption-information field. */
246 uint32_t uEntryIntrInfo;
247 /** The VM-entry exception error code field. */
248 uint32_t uEntryXcptErrorCode;
249 /** The VM-entry instruction length field. */
250 uint32_t cbEntryInstr;
251
252 /** IDT-vectoring information field. */
253 uint32_t uIdtVectoringInfo;
254 /** IDT-vectoring error code. */
255 uint32_t uIdtVectoringErrorCode;
256
257 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
258 uint32_t fVmcsFieldsRead;
259 /** Whether TSC-offsetting should be setup before VM-entry. */
260 bool fUpdateTscOffsettingAndPreemptTimer;
261 /** Whether the VM-exit was caused by a page-fault during delivery of a
262 * contributory exception or a page-fault. */
263 bool fVectoringPF;
264 /** Whether the guest FPU was active at the time of VM-exit. */
265 bool fWasGuestFPUStateActive;
266 /** Whether the guest debug state was active at the time of VM-exit. */
267 bool fWasGuestDebugStateActive;
268 /** Whether the hyper debug state was active at the time of VM-exit. */
269 bool fWasHyperDebugStateActive;
270} VMXTRANSIENT;
271AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
272AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
273AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
274AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
275/** Pointer to VMX transient state. */
276typedef VMXTRANSIENT *PVMXTRANSIENT;
277
278
279/**
280 * MSR-bitmap read permissions.
281 */
282typedef enum VMXMSREXITREAD
283{
284 /** Reading this MSR causes a VM-exit. */
285 VMXMSREXIT_INTERCEPT_READ = 0xb,
286 /** Reading this MSR does not cause a VM-exit. */
287 VMXMSREXIT_PASSTHRU_READ
288} VMXMSREXITREAD;
289
290/**
291 * MSR-bitmap write permissions.
292 */
293typedef enum VMXMSREXITWRITE
294{
295 /** Writing to this MSR causes a VM-exit. */
296 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
297 /** Writing to this MSR does not cause a VM-exit. */
298 VMXMSREXIT_PASSTHRU_WRITE
299} VMXMSREXITWRITE;
300
301/**
302 * VMX VM-exit handler.
303 *
304 * @returns VBox status code.
305 * @param pVCpu Pointer to the VMCPU.
306 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
307 * out-of-sync. Make sure to update the required
308 * fields before using them.
309 * @param pVmxTransient Pointer to the VMX-transient structure.
310 */
311#ifndef HMVMX_USE_FUNCTION_TABLE
312typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313#else
314typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
315/** Pointer to VM-exit handler. */
316typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
317#endif
318
319
320/*******************************************************************************
321* Internal Functions *
322*******************************************************************************/
323static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
324static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
325static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
326static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
327 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
328#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
329static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
330#endif
331#ifndef HMVMX_USE_FUNCTION_TABLE
332DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
333# define HMVMX_EXIT_DECL static int
334#else
335# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
336#endif
337
338/** @name VM-exit handlers.
339 * @{
340 */
341static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
342static FNVMXEXITHANDLER hmR0VmxExitExtInt;
343static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
344static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
345static FNVMXEXITHANDLER hmR0VmxExitSipi;
346static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
347static FNVMXEXITHANDLER hmR0VmxExitSmi;
348static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
349static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
350static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
351static FNVMXEXITHANDLER hmR0VmxExitCpuid;
352static FNVMXEXITHANDLER hmR0VmxExitGetsec;
353static FNVMXEXITHANDLER hmR0VmxExitHlt;
354static FNVMXEXITHANDLER hmR0VmxExitInvd;
355static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
356static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
357static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
358static FNVMXEXITHANDLER hmR0VmxExitRsm;
359static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
360static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
361static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
362static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
363static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
364static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
365static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
366static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
367static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
368static FNVMXEXITHANDLER hmR0VmxExitMwait;
369static FNVMXEXITHANDLER hmR0VmxExitMtf;
370static FNVMXEXITHANDLER hmR0VmxExitMonitor;
371static FNVMXEXITHANDLER hmR0VmxExitPause;
372static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
373static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
374static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
375static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
376static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
377static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
378static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
379static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
380static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
381static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
382static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
383static FNVMXEXITHANDLER hmR0VmxExitRdrand;
384static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
385/** @} */
386
387static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
388static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
389static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
390static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
391static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
392static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
393static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
394static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
395
396/*******************************************************************************
397* Global Variables *
398*******************************************************************************/
399#ifdef HMVMX_USE_FUNCTION_TABLE
400
401/**
402 * VMX_EXIT dispatch table.
403 */
404static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
405{
406 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
407 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
408 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
409 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
410 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
411 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
412 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
413 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
414 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
415 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
416 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
417 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
418 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
419 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
420 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
421 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
422 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
423 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
424 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
425 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
426 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
427 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
428 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
429 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
430 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
431 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
432 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
433 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
434 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
435 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
436 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
437 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
438 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
439 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
440 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
441 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
442 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
443 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
444 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
445 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
446 /* 40 UNDEFINED */ hmR0VmxExitPause,
447 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
448 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
449 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
450 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
451 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
452 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
453 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
454 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
455 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
456 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
457 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
458 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
459 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
460 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
461 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
462 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
463 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
464 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
465 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
466};
467#endif /* HMVMX_USE_FUNCTION_TABLE */
468
469#ifdef VBOX_STRICT
470static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
471{
472 /* 0 */ "(Not Used)",
473 /* 1 */ "VMCALL executed in VMX root operation.",
474 /* 2 */ "VMCLEAR with invalid physical address.",
475 /* 3 */ "VMCLEAR with VMXON pointer.",
476 /* 4 */ "VMLAUNCH with non-clear VMCS.",
477 /* 5 */ "VMRESUME with non-launched VMCS.",
478 /* 6 */ "VMRESUME after VMXOFF",
479 /* 7 */ "VM entry with invalid control fields.",
480 /* 8 */ "VM entry with invalid host state fields.",
481 /* 9 */ "VMPTRLD with invalid physical address.",
482 /* 10 */ "VMPTRLD with VMXON pointer.",
483 /* 11 */ "VMPTRLD with incorrect revision identifier.",
484 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
485 /* 13 */ "VMWRITE to read-only VMCS component.",
486 /* 14 */ "(Not Used)",
487 /* 15 */ "VMXON executed in VMX root operation.",
488 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
489 /* 17 */ "VM entry with non-launched executing VMCS.",
490 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
491 /* 19 */ "VMCALL with non-clear VMCS.",
492 /* 20 */ "VMCALL with invalid VM-exit control fields.",
493 /* 21 */ "(Not Used)",
494 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
495 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
496 /* 24 */ "VMCALL with invalid SMM-monitor features.",
497 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
498 /* 26 */ "VM entry with events blocked by MOV SS.",
499 /* 27 */ "(Not Used)",
500 /* 28 */ "Invalid operand to INVEPT/INVVPID."
501};
502#endif /* VBOX_STRICT */
503
504
505
506/**
507 * Updates the VM's last error record. If there was a VMX instruction error,
508 * reads the error data from the VMCS and updates VCPU's last error record as
509 * well.
510 *
511 * @param pVM Pointer to the VM.
512 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
513 * VERR_VMX_UNABLE_TO_START_VM or
514 * VERR_VMX_INVALID_VMCS_FIELD).
515 * @param rc The error code.
516 */
517static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
518{
519 AssertPtr(pVM);
520 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
521 || rc == VERR_VMX_UNABLE_TO_START_VM)
522 {
523 AssertPtrReturnVoid(pVCpu);
524 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
525 }
526 pVM->hm.s.lLastError = rc;
527}
528
529
530/**
531 * Reads the VM-entry interruption-information field from the VMCS into the VMX
532 * transient structure.
533 *
534 * @returns VBox status code.
535 * @param pVmxTransient Pointer to the VMX transient structure.
536 *
537 * @remarks No-long-jump zone!!!
538 */
539DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
540{
541 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
542 AssertRCReturn(rc, rc);
543 return VINF_SUCCESS;
544}
545
546
547/**
548 * Reads the VM-entry exception error code field from the VMCS into
549 * the VMX transient structure.
550 *
551 * @returns VBox status code.
552 * @param pVmxTransient Pointer to the VMX transient structure.
553 *
554 * @remarks No-long-jump zone!!!
555 */
556DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
557{
558 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
559 AssertRCReturn(rc, rc);
560 return VINF_SUCCESS;
561}
562
563
564/**
565 * Reads the VM-entry exception error code field from the VMCS into
566 * the VMX transient structure.
567 *
568 * @returns VBox status code.
569 * @param pVCpu Pointer to the VMCPU.
570 * @param pVmxTransient Pointer to the VMX transient structure.
571 *
572 * @remarks No-long-jump zone!!!
573 */
574DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
575{
576 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
577 AssertRCReturn(rc, rc);
578 return VINF_SUCCESS;
579}
580
581
582/**
583 * Reads the VM-exit interruption-information field from the VMCS into the VMX
584 * transient structure.
585 *
586 * @returns VBox status code.
587 * @param pVCpu Pointer to the VMCPU.
588 * @param pVmxTransient Pointer to the VMX transient structure.
589 */
590DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
591{
592 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
593 {
594 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
595 AssertRCReturn(rc, rc);
596 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
597 }
598 return VINF_SUCCESS;
599}
600
601
602/**
603 * Reads the VM-exit interruption error code from the VMCS into the VMX
604 * transient structure.
605 *
606 * @returns VBox status code.
607 * @param pVCpu Pointer to the VMCPU.
608 * @param pVmxTransient Pointer to the VMX transient structure.
609 */
610DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
611{
612 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
613 {
614 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
615 AssertRCReturn(rc, rc);
616 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
617 }
618 return VINF_SUCCESS;
619}
620
621
622/**
623 * Reads the VM-exit instruction length field from the VMCS into the VMX
624 * transient structure.
625 *
626 * @returns VBox status code.
627 * @param pVCpu Pointer to the VMCPU.
628 * @param pVmxTransient Pointer to the VMX transient structure.
629 */
630DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
631{
632 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
633 {
634 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
635 AssertRCReturn(rc, rc);
636 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
637 }
638 return VINF_SUCCESS;
639}
640
641
642/**
643 * Reads the VM-exit instruction-information field from the VMCS into
644 * the VMX transient structure.
645 *
646 * @returns VBox status code.
647 * @param pVCpu The cross context per CPU structure.
648 * @param pVmxTransient Pointer to the VMX transient structure.
649 */
650DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
651{
652 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
653 {
654 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
655 AssertRCReturn(rc, rc);
656 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
657 }
658 return VINF_SUCCESS;
659}
660
661
662/**
663 * Reads the exit qualification from the VMCS into the VMX transient structure.
664 *
665 * @returns VBox status code.
666 * @param pVCpu Pointer to the VMCPU.
667 * @param pVmxTransient Pointer to the VMX transient structure.
668 */
669DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
670{
671 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
672 {
673 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
674 AssertRCReturn(rc, rc);
675 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
676 }
677 return VINF_SUCCESS;
678}
679
680
681/**
682 * Reads the IDT-vectoring information field from the VMCS into the VMX
683 * transient structure.
684 *
685 * @returns VBox status code.
686 * @param pVmxTransient Pointer to the VMX transient structure.
687 *
688 * @remarks No-long-jump zone!!!
689 */
690DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
691{
692 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
693 {
694 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
695 AssertRCReturn(rc, rc);
696 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
697 }
698 return VINF_SUCCESS;
699}
700
701
702/**
703 * Reads the IDT-vectoring error code from the VMCS into the VMX
704 * transient structure.
705 *
706 * @returns VBox status code.
707 * @param pVmxTransient Pointer to the VMX transient structure.
708 */
709DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
710{
711 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
712 {
713 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
714 AssertRCReturn(rc, rc);
715 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
716 }
717 return VINF_SUCCESS;
718}
719
720
721/**
722 * Enters VMX root mode operation on the current CPU.
723 *
724 * @returns VBox status code.
725 * @param pVM Pointer to the VM (optional, can be NULL, after
726 * a resume).
727 * @param HCPhysCpuPage Physical address of the VMXON region.
728 * @param pvCpuPage Pointer to the VMXON region.
729 */
730static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
731{
732 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
733 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
735
736 if (pVM)
737 {
738 /* Write the VMCS revision dword to the VMXON region. */
739 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
740 }
741
742 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
743 RTCCUINTREG uEflags = ASMIntDisableFlags();
744
745 /* Enable the VMX bit in CR4 if necessary. */
746 RTCCUINTREG uCr4 = ASMGetCR4();
747 if (!(uCr4 & X86_CR4_VMXE))
748 ASMSetCR4(uCr4 | X86_CR4_VMXE);
749
750 /* Enter VMX root mode. */
751 int rc = VMXEnable(HCPhysCpuPage);
752 if (RT_FAILURE(rc))
753 ASMSetCR4(uCr4);
754
755 /* Restore interrupts. */
756 ASMSetFlags(uEflags);
757 return rc;
758}
759
760
761/**
762 * Exits VMX root mode operation on the current CPU.
763 *
764 * @returns VBox status code.
765 */
766static int hmR0VmxLeaveRootMode(void)
767{
768 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
769
770 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
771 RTCCUINTREG uEflags = ASMIntDisableFlags();
772
773 /* If we're for some reason not in VMX root mode, then don't leave it. */
774 RTCCUINTREG uHostCR4 = ASMGetCR4();
775
776 int rc;
777 if (uHostCR4 & X86_CR4_VMXE)
778 {
779 /* Exit VMX root mode and clear the VMX bit in CR4. */
780 VMXDisable();
781 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
782 rc = VINF_SUCCESS;
783 }
784 else
785 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
786
787 /* Restore interrupts. */
788 ASMSetFlags(uEflags);
789 return rc;
790}
791
792
793/**
794 * Allocates and maps one physically contiguous page. The allocated page is
795 * zero'd out. (Used by various VT-x structures).
796 *
797 * @returns IPRT status code.
798 * @param pMemObj Pointer to the ring-0 memory object.
799 * @param ppVirt Where to store the virtual address of the
800 * allocation.
801 * @param pPhys Where to store the physical address of the
802 * allocation.
803 */
804DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
805{
806 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
807 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
808 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
809
810 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
811 if (RT_FAILURE(rc))
812 return rc;
813 *ppVirt = RTR0MemObjAddress(*pMemObj);
814 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
815 ASMMemZero32(*ppVirt, PAGE_SIZE);
816 return VINF_SUCCESS;
817}
818
819
820/**
821 * Frees and unmaps an allocated physical page.
822 *
823 * @param pMemObj Pointer to the ring-0 memory object.
824 * @param ppVirt Where to re-initialize the virtual address of
825 * allocation as 0.
826 * @param pHCPhys Where to re-initialize the physical address of the
827 * allocation as 0.
828 */
829DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
830{
831 AssertPtr(pMemObj);
832 AssertPtr(ppVirt);
833 AssertPtr(pHCPhys);
834 if (*pMemObj != NIL_RTR0MEMOBJ)
835 {
836 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
837 AssertRC(rc);
838 *pMemObj = NIL_RTR0MEMOBJ;
839 *ppVirt = 0;
840 *pHCPhys = 0;
841 }
842}
843
844
845/**
846 * Worker function to free VT-x related structures.
847 *
848 * @returns IPRT status code.
849 * @param pVM Pointer to the VM.
850 */
851static void hmR0VmxStructsFree(PVM pVM)
852{
853 for (VMCPUID i = 0; i < pVM->cCpus; i++)
854 {
855 PVMCPU pVCpu = &pVM->aCpus[i];
856 AssertPtr(pVCpu);
857
858#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
859 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
860 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
861#endif
862
863 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
864 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
865
866 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
867 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
868 }
869
870 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
871#ifdef VBOX_WITH_CRASHDUMP_MAGIC
872 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
873#endif
874}
875
876
877/**
878 * Worker function to allocate VT-x related VM structures.
879 *
880 * @returns IPRT status code.
881 * @param pVM Pointer to the VM.
882 */
883static int hmR0VmxStructsAlloc(PVM pVM)
884{
885 /*
886 * Initialize members up-front so we can cleanup properly on allocation failure.
887 */
888#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
889 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
890 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
891 pVM->hm.s.vmx.HCPhys##a_Name = 0;
892
893#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
894 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
895 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
896 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
897
898#ifdef VBOX_WITH_CRASHDUMP_MAGIC
899 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
900#endif
901 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
902
903 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
904 for (VMCPUID i = 0; i < pVM->cCpus; i++)
905 {
906 PVMCPU pVCpu = &pVM->aCpus[i];
907 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
908 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
909 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
910#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
911 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
912 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
913#endif
914 }
915#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
916#undef VMXLOCAL_INIT_VM_MEMOBJ
917
918 /*
919 * Allocate all the VT-x structures.
920 */
921 int rc = VINF_SUCCESS;
922#ifdef VBOX_WITH_CRASHDUMP_MAGIC
923 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
924 if (RT_FAILURE(rc))
925 goto cleanup;
926 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
927 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
928#endif
929
930 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
931 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
932 {
933 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
934 &pVM->hm.s.vmx.HCPhysApicAccess);
935 if (RT_FAILURE(rc))
936 goto cleanup;
937 }
938
939 /*
940 * Initialize per-VCPU VT-x structures.
941 */
942 for (VMCPUID i = 0; i < pVM->cCpus; i++)
943 {
944 PVMCPU pVCpu = &pVM->aCpus[i];
945 AssertPtr(pVCpu);
946
947 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
948 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
949 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
950 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
951
952 /* Allocate the VM control structure (VMCS). */
953 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956
957 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
958 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
959 {
960 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
961 &pVCpu->hm.s.vmx.HCPhysVirtApic);
962 if (RT_FAILURE(rc))
963 goto cleanup;
964 }
965
966 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
967 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
968 {
969 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
970 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
971 if (RT_FAILURE(rc))
972 goto cleanup;
973 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
974 }
975
976#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
977 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
978 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
979 if (RT_FAILURE(rc))
980 goto cleanup;
981
982 /* Allocate the VM-exit MSR-load page for the host MSRs. */
983 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
984 if (RT_FAILURE(rc))
985 goto cleanup;
986#endif
987 }
988
989 return VINF_SUCCESS;
990
991cleanup:
992 hmR0VmxStructsFree(pVM);
993 return rc;
994}
995
996
997/**
998 * Does global VT-x initialization (called during module initialization).
999 *
1000 * @returns VBox status code.
1001 */
1002VMMR0DECL(int) VMXR0GlobalInit(void)
1003{
1004#ifdef HMVMX_USE_FUNCTION_TABLE
1005 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1006# ifdef VBOX_STRICT
1007 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1008 Assert(g_apfnVMExitHandlers[i]);
1009# endif
1010#endif
1011 return VINF_SUCCESS;
1012}
1013
1014
1015/**
1016 * Does global VT-x termination (called during module termination).
1017 */
1018VMMR0DECL(void) VMXR0GlobalTerm()
1019{
1020 /* Nothing to do currently. */
1021}
1022
1023
1024/**
1025 * Sets up and activates VT-x on the current CPU.
1026 *
1027 * @returns VBox status code.
1028 * @param pCpu Pointer to the global CPU info struct.
1029 * @param pVM Pointer to the VM (can be NULL after a host resume
1030 * operation).
1031 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1032 * fEnabledByHost is true).
1033 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1034 * @a fEnabledByHost is true).
1035 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1036 * enable VT-x on the host.
1037 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1038 */
1039VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1040 void *pvMsrs)
1041{
1042 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1043 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1044 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1045
1046 /* Enable VT-x if it's not already enabled by the host. */
1047 if (!fEnabledByHost)
1048 {
1049 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1050 if (RT_FAILURE(rc))
1051 return rc;
1052 }
1053
1054 /*
1055 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1056 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1057 */
1058 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1059 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1060 {
1061 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1062 pCpu->fFlushAsidBeforeUse = false;
1063 }
1064 else
1065 pCpu->fFlushAsidBeforeUse = true;
1066
1067 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1068 ++pCpu->cTlbFlushes;
1069
1070 return VINF_SUCCESS;
1071}
1072
1073
1074/**
1075 * Deactivates VT-x on the current CPU.
1076 *
1077 * @returns VBox status code.
1078 * @param pCpu Pointer to the global CPU info struct.
1079 * @param pvCpuPage Pointer to the VMXON region.
1080 * @param HCPhysCpuPage Physical address of the VMXON region.
1081 *
1082 * @remarks This function should never be called when SUPR0EnableVTx() or
1083 * similar was used to enable VT-x on the host.
1084 */
1085VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1086{
1087 NOREF(pCpu);
1088 NOREF(pvCpuPage);
1089 NOREF(HCPhysCpuPage);
1090
1091 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1092 return hmR0VmxLeaveRootMode();
1093}
1094
1095
1096/**
1097 * Sets the permission bits for the specified MSR in the MSR bitmap.
1098 *
1099 * @param pVCpu Pointer to the VMCPU.
1100 * @param uMSR The MSR value.
1101 * @param enmRead Whether reading this MSR causes a VM-exit.
1102 * @param enmWrite Whether writing this MSR causes a VM-exit.
1103 */
1104static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1105{
1106 int32_t iBit;
1107 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1108
1109 /*
1110 * Layout:
1111 * 0x000 - 0x3ff - Low MSR read bits
1112 * 0x400 - 0x7ff - High MSR read bits
1113 * 0x800 - 0xbff - Low MSR write bits
1114 * 0xc00 - 0xfff - High MSR write bits
1115 */
1116 if (uMsr <= 0x00001FFF)
1117 iBit = uMsr;
1118 else if ( uMsr >= 0xC0000000
1119 && uMsr <= 0xC0001FFF)
1120 {
1121 iBit = (uMsr - 0xC0000000);
1122 pbMsrBitmap += 0x400;
1123 }
1124 else
1125 {
1126 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1127 return;
1128 }
1129
1130 Assert(iBit <= 0x1fff);
1131 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1132 ASMBitSet(pbMsrBitmap, iBit);
1133 else
1134 ASMBitClear(pbMsrBitmap, iBit);
1135
1136 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1137 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1138 else
1139 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1140}
1141
1142
1143/**
1144 * Flushes the TLB using EPT.
1145 *
1146 * @returns VBox status code.
1147 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1148 * enmFlush).
1149 * @param enmFlush Type of flush.
1150 *
1151 * @remarks Caller is responsible for making sure this function is called only
1152 * when NestedPaging is supported and providing @a enmFlush that is
1153 * supported by the CPU.
1154 */
1155static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1156{
1157 uint64_t au64Descriptor[2];
1158 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1159 au64Descriptor[0] = 0;
1160 else
1161 {
1162 Assert(pVCpu);
1163 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1164 }
1165 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1166
1167 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1168 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1169 rc));
1170 if ( RT_SUCCESS(rc)
1171 && pVCpu)
1172 {
1173 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1174 }
1175}
1176
1177
1178/**
1179 * Flushes the TLB using VPID.
1180 *
1181 * @returns VBox status code.
1182 * @param pVM Pointer to the VM.
1183 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1184 * enmFlush).
1185 * @param enmFlush Type of flush.
1186 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1187 * on @a enmFlush).
1188 */
1189static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1190{
1191 AssertPtr(pVM);
1192 Assert(pVM->hm.s.vmx.fVpid);
1193
1194 uint64_t au64Descriptor[2];
1195 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1196 {
1197 au64Descriptor[0] = 0;
1198 au64Descriptor[1] = 0;
1199 }
1200 else
1201 {
1202 AssertPtr(pVCpu);
1203 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1204 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1205 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1206 au64Descriptor[1] = GCPtr;
1207 }
1208
1209 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1210 AssertMsg(rc == VINF_SUCCESS,
1211 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1212 if ( RT_SUCCESS(rc)
1213 && pVCpu)
1214 {
1215 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1216 }
1217}
1218
1219
1220/**
1221 * Invalidates a guest page by guest virtual address. Only relevant for
1222 * EPT/VPID, otherwise there is nothing really to invalidate.
1223 *
1224 * @returns VBox status code.
1225 * @param pVM Pointer to the VM.
1226 * @param pVCpu Pointer to the VMCPU.
1227 * @param GCVirt Guest virtual address of the page to invalidate.
1228 */
1229VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1230{
1231 AssertPtr(pVM);
1232 AssertPtr(pVCpu);
1233 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1234
1235 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1236 if (!fFlushPending)
1237 {
1238 /*
1239 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1240 * See @bugref{6043} and @bugref{6177}.
1241 *
1242 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1243 * function maybe called in a loop with individual addresses.
1244 */
1245 if (pVM->hm.s.vmx.fVpid)
1246 {
1247 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1248 {
1249 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1250 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1251 }
1252 else
1253 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1254 }
1255 else if (pVM->hm.s.fNestedPaging)
1256 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1257 }
1258
1259 return VINF_SUCCESS;
1260}
1261
1262
1263/**
1264 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1265 * otherwise there is nothing really to invalidate.
1266 *
1267 * @returns VBox status code.
1268 * @param pVM Pointer to the VM.
1269 * @param pVCpu Pointer to the VMCPU.
1270 * @param GCPhys Guest physical address of the page to invalidate.
1271 */
1272VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1273{
1274 LogFlowFunc(("%RGp\n", GCPhys));
1275
1276 /*
1277 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1278 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1279 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1280 */
1281 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1282 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1283 return VINF_SUCCESS;
1284}
1285
1286
1287/**
1288 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1289 * case where neither EPT nor VPID is supported by the CPU.
1290 *
1291 * @param pVM Pointer to the VM.
1292 * @param pVCpu Pointer to the VMCPU.
1293 * @param pCpu Pointer to the global HM struct.
1294 *
1295 * @remarks Called with interrupts disabled.
1296 */
1297static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1298{
1299 AssertPtr(pVCpu);
1300 AssertPtr(pCpu);
1301 NOREF(pVM);
1302
1303 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1304 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1305
1306 pVCpu->hm.s.TlbShootdown.cPages = 0;
1307 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1308 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1309 pVCpu->hm.s.fForceTLBFlush = false;
1310 return;
1311}
1312
1313
1314/**
1315 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1316 *
1317 * @param pVM Pointer to the VM.
1318 * @param pVCpu Pointer to the VMCPU.
1319 * @param pCpu Pointer to the global HM CPU struct.
1320 * @remarks All references to "ASID" in this function pertains to "VPID" in
1321 * Intel's nomenclature. The reason is, to avoid confusion in compare
1322 * statements since the host-CPU copies are named "ASID".
1323 *
1324 * @remarks Called with interrupts disabled.
1325 */
1326static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1327{
1328#ifdef VBOX_WITH_STATISTICS
1329 bool fTlbFlushed = false;
1330# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1331# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1332 if (!fTlbFlushed) \
1333 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1334 } while (0)
1335#else
1336# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1337# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1338#endif
1339
1340 AssertPtr(pVM);
1341 AssertPtr(pCpu);
1342 AssertPtr(pVCpu);
1343 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1344 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1345 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1346
1347
1348 /*
1349 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1350 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1351 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1352 */
1353 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1354 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1355 {
1356 ++pCpu->uCurrentAsid;
1357 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1358 {
1359 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1360 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1361 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1362 }
1363
1364 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1365 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1366 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1367
1368 /*
1369 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1370 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1371 */
1372 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1373 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1374 HMVMX_SET_TAGGED_TLB_FLUSHED();
1375 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1376 }
1377
1378 /* Check for explicit TLB shootdowns. */
1379 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1380 {
1381 /*
1382 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1383 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1384 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1385 * but not guest-physical mappings.
1386 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1387 */
1388 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1389 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1390 HMVMX_SET_TAGGED_TLB_FLUSHED();
1391 }
1392
1393 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1394 * not be executed. See hmQueueInvlPage() where it is commented
1395 * out. Support individual entry flushing someday. */
1396 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1397 {
1398 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1399
1400 /*
1401 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1402 * as supported by the CPU.
1403 */
1404 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1405 {
1406 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1407 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1408 }
1409 else
1410 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1411
1412 HMVMX_SET_TAGGED_TLB_FLUSHED();
1413 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1414 }
1415
1416 pVCpu->hm.s.TlbShootdown.cPages = 0;
1417 pVCpu->hm.s.fForceTLBFlush = false;
1418
1419 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1420
1421 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1422 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1423 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1424 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1425 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1426 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1427 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1428 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1429
1430 /* Update VMCS with the VPID. */
1431 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1432 AssertRC(rc);
1433
1434#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1435}
1436
1437
1438/**
1439 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1440 *
1441 * @returns VBox status code.
1442 * @param pVM Pointer to the VM.
1443 * @param pVCpu Pointer to the VMCPU.
1444 * @param pCpu Pointer to the global HM CPU struct.
1445 *
1446 * @remarks Called with interrupts disabled.
1447 */
1448static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1449{
1450 AssertPtr(pVM);
1451 AssertPtr(pVCpu);
1452 AssertPtr(pCpu);
1453 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1454 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1455
1456 /*
1457 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1458 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1459 */
1460 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1461 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1462 {
1463 pVCpu->hm.s.fForceTLBFlush = true;
1464 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1465 }
1466
1467 /* Check for explicit TLB shootdown flushes. */
1468 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1469 {
1470 pVCpu->hm.s.fForceTLBFlush = true;
1471 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1472 }
1473
1474 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1475 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1476
1477 if (pVCpu->hm.s.fForceTLBFlush)
1478 {
1479 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1480 pVCpu->hm.s.fForceTLBFlush = false;
1481 }
1482 else
1483 {
1484 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1485 * not be executed. See hmQueueInvlPage() where it is commented
1486 * out. Support individual entry flushing someday. */
1487 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1488 {
1489 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1490 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1491 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1492 }
1493 else
1494 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1495 }
1496
1497 pVCpu->hm.s.TlbShootdown.cPages = 0;
1498 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1499}
1500
1501
1502/**
1503 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1504 *
1505 * @returns VBox status code.
1506 * @param pVM Pointer to the VM.
1507 * @param pVCpu Pointer to the VMCPU.
1508 * @param pCpu Pointer to the global HM CPU struct.
1509 *
1510 * @remarks Called with interrupts disabled.
1511 */
1512static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1513{
1514 AssertPtr(pVM);
1515 AssertPtr(pVCpu);
1516 AssertPtr(pCpu);
1517 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1518 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1519
1520 /*
1521 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1522 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1523 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1524 */
1525 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1526 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1527 {
1528 pVCpu->hm.s.fForceTLBFlush = true;
1529 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1530 }
1531
1532 /* Check for explicit TLB shootdown flushes. */
1533 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1534 {
1535 /*
1536 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1537 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1538 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1539 */
1540 pVCpu->hm.s.fForceTLBFlush = true;
1541 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1542 }
1543
1544 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1545 if (pVCpu->hm.s.fForceTLBFlush)
1546 {
1547 ++pCpu->uCurrentAsid;
1548 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1549 {
1550 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1551 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1552 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1553 }
1554
1555 pVCpu->hm.s.fForceTLBFlush = false;
1556 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1557 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1558 if (pCpu->fFlushAsidBeforeUse)
1559 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1560 }
1561 else
1562 {
1563 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1564 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1565 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1566 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1567
1568 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1569 * not be executed. See hmQueueInvlPage() where it is commented
1570 * out. Support individual entry flushing someday. */
1571 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1572 {
1573 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1574 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1575 {
1576 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1577 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1578 }
1579 else
1580 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1581 }
1582 else
1583 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1584 }
1585
1586 pVCpu->hm.s.TlbShootdown.cPages = 0;
1587 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1588
1589 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1590 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1591 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1592 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1593 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1594 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1595
1596 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1597 AssertRC(rc);
1598}
1599
1600
1601/**
1602 * Flushes the guest TLB entry based on CPU capabilities.
1603 *
1604 * @param pVCpu Pointer to the VMCPU.
1605 * @param pCpu Pointer to the global HM CPU struct.
1606 */
1607DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1608{
1609 PVM pVM = pVCpu->CTX_SUFF(pVM);
1610 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1611 {
1612 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1613 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1614 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1615 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1616 default:
1617 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1618 break;
1619 }
1620}
1621
1622
1623/**
1624 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1625 * TLB entries from the host TLB before VM-entry.
1626 *
1627 * @returns VBox status code.
1628 * @param pVM Pointer to the VM.
1629 */
1630static int hmR0VmxSetupTaggedTlb(PVM pVM)
1631{
1632 /*
1633 * Determine optimal flush type for Nested Paging.
1634 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1635 * guest execution (see hmR3InitFinalizeR0()).
1636 */
1637 if (pVM->hm.s.fNestedPaging)
1638 {
1639 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1640 {
1641 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1642 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1643 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1644 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1645 else
1646 {
1647 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1648 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1649 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1650 }
1651
1652 /* Make sure the write-back cacheable memory type for EPT is supported. */
1653 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1654 {
1655 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1656 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1657 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1658 }
1659 }
1660 else
1661 {
1662 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1663 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1664 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1665 }
1666 }
1667
1668 /*
1669 * Determine optimal flush type for VPID.
1670 */
1671 if (pVM->hm.s.vmx.fVpid)
1672 {
1673 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1674 {
1675 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1676 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1677 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1678 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1679 else
1680 {
1681 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1682 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1683 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1684 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1685 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1686 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1687 pVM->hm.s.vmx.fVpid = false;
1688 }
1689 }
1690 else
1691 {
1692 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1693 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1694 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1695 pVM->hm.s.vmx.fVpid = false;
1696 }
1697 }
1698
1699 /*
1700 * Setup the handler for flushing tagged-TLBs.
1701 */
1702 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1703 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1704 else if (pVM->hm.s.fNestedPaging)
1705 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1706 else if (pVM->hm.s.vmx.fVpid)
1707 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1708 else
1709 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1710 return VINF_SUCCESS;
1711}
1712
1713
1714/**
1715 * Sets up pin-based VM-execution controls in the VMCS.
1716 *
1717 * @returns VBox status code.
1718 * @param pVM Pointer to the VM.
1719 * @param pVCpu Pointer to the VMCPU.
1720 */
1721static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1722{
1723 AssertPtr(pVM);
1724 AssertPtr(pVCpu);
1725
1726 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1727 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1728
1729 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1730 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1731 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1732
1733 /* Enable the VMX preemption timer. */
1734 if (pVM->hm.s.vmx.fUsePreemptTimer)
1735 {
1736 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1737 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1738 }
1739
1740 if ((val & zap) != val)
1741 {
1742 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1743 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1744 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1745 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1746 }
1747
1748 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1749 AssertRCReturn(rc, rc);
1750
1751 /* Update VCPU with the currently set pin-based VM-execution controls. */
1752 pVCpu->hm.s.vmx.u32PinCtls = val;
1753 return rc;
1754}
1755
1756
1757/**
1758 * Sets up processor-based VM-execution controls in the VMCS.
1759 *
1760 * @returns VBox status code.
1761 * @param pVM Pointer to the VM.
1762 * @param pVMCPU Pointer to the VMCPU.
1763 */
1764static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1765{
1766 AssertPtr(pVM);
1767 AssertPtr(pVCpu);
1768
1769 int rc = VERR_INTERNAL_ERROR_5;
1770 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1771 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1772
1773 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1774 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1775 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1776 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1777 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1778 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1779 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1780
1781 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1782 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1783 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1784 {
1785 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1786 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1787 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1788 }
1789
1790 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1791 if (!pVM->hm.s.fNestedPaging)
1792 {
1793 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1794 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1795 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1796 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1797 }
1798
1799 /* Use TPR shadowing if supported by the CPU. */
1800 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1801 {
1802 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1803 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1804 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1805 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1806 AssertRCReturn(rc, rc);
1807
1808 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1809 /* CR8 writes causes a VM-exit based on TPR threshold. */
1810 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1811 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1812 }
1813 else
1814 {
1815 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1816 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1817 }
1818
1819 /* Use MSR-bitmaps if supported by the CPU. */
1820 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1821 {
1822 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1823
1824 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1825 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1826 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1827 AssertRCReturn(rc, rc);
1828
1829 /*
1830 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1831 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1832 */
1833 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1834 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1835 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1836 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1837 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1838 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1839 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1840 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1841 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1842 }
1843
1844 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1845 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1846 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1847
1848 if ((val & zap) != val)
1849 {
1850 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1851 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1852 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1853 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1854 }
1855
1856 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1857 AssertRCReturn(rc, rc);
1858
1859 /* Update VCPU with the currently set processor-based VM-execution controls. */
1860 pVCpu->hm.s.vmx.u32ProcCtls = val;
1861
1862 /*
1863 * Secondary processor-based VM-execution controls.
1864 */
1865 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1866 {
1867 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1868 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1869
1870 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1871 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1872
1873 if (pVM->hm.s.fNestedPaging)
1874 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1875 else
1876 {
1877 /*
1878 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1879 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1880 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1881 */
1882 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1883 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1884 }
1885
1886 if (pVM->hm.s.vmx.fVpid)
1887 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1888
1889 if (pVM->hm.s.vmx.fUnrestrictedGuest)
1890 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1891
1892 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1893 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1894 * done dynamically. */
1895 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1896 {
1897 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1898 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1899 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1900 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1901 AssertRCReturn(rc, rc);
1902 }
1903
1904 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1905 {
1906 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1907 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1908 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1909 }
1910
1911 if ((val & zap) != val)
1912 {
1913 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1914 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1915 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1916 }
1917
1918 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1919 AssertRCReturn(rc, rc);
1920
1921 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1922 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1923 }
1924 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1925 {
1926 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1927 "available\n"));
1928 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1929 }
1930
1931 return VINF_SUCCESS;
1932}
1933
1934
1935/**
1936 * Sets up miscellaneous (everything other than Pin & Processor-based
1937 * VM-execution) control fields in the VMCS.
1938 *
1939 * @returns VBox status code.
1940 * @param pVM Pointer to the VM.
1941 * @param pVCpu Pointer to the VMCPU.
1942 */
1943static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1944{
1945 AssertPtr(pVM);
1946 AssertPtr(pVCpu);
1947
1948 int rc = VERR_GENERAL_FAILURE;
1949
1950 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1951#if 0
1952 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
1953 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1954 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1955
1956 /*
1957 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1958 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1959 * We thus use the exception bitmap to control it rather than use both.
1960 */
1961 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1962 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1963
1964 /** @todo Explore possibility of using IO-bitmaps. */
1965 /* All IO & IOIO instructions cause VM-exits. */
1966 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1967 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1968
1969 /* Initialize the MSR-bitmap area. */
1970 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1971 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1972 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1973#endif
1974
1975#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1976 /* Setup MSR autoloading/storing. */
1977 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1978 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1979 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1980 AssertRCReturn(rc, rc);
1981 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1982 AssertRCReturn(rc, rc);
1983
1984 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1985 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1986 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1987 AssertRCReturn(rc, rc);
1988#endif
1989
1990 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1991 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1992 AssertRCReturn(rc, rc);
1993
1994 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1995#if 0
1996 /* Setup debug controls */
1997 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1998 AssertRCReturn(rc, rc);
1999 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2000 AssertRCReturn(rc, rc);
2001#endif
2002
2003 return rc;
2004}
2005
2006
2007/**
2008 * Sets up the initial exception bitmap in the VMCS based on static conditions
2009 * (i.e. conditions that cannot ever change after starting the VM).
2010 *
2011 * @returns VBox status code.
2012 * @param pVM Pointer to the VM.
2013 * @param pVCpu Pointer to the VMCPU.
2014 */
2015static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2016{
2017 AssertPtr(pVM);
2018 AssertPtr(pVCpu);
2019
2020 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2021
2022 uint32_t u32XcptBitmap = 0;
2023
2024 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2025 if (!pVM->hm.s.fNestedPaging)
2026 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2027
2028 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2029 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2030 AssertRCReturn(rc, rc);
2031 return rc;
2032}
2033
2034
2035/**
2036 * Sets up the initial guest-state mask. The guest-state mask is consulted
2037 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2038 * for the nested virtualization case (as it would cause a VM-exit).
2039 *
2040 * @param pVCpu Pointer to the VMCPU.
2041 */
2042static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2043{
2044 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2045 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2046 return VINF_SUCCESS;
2047}
2048
2049
2050/**
2051 * Does per-VM VT-x initialization.
2052 *
2053 * @returns VBox status code.
2054 * @param pVM Pointer to the VM.
2055 */
2056VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2057{
2058 LogFlowFunc(("pVM=%p\n", pVM));
2059
2060 int rc = hmR0VmxStructsAlloc(pVM);
2061 if (RT_FAILURE(rc))
2062 {
2063 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2064 return rc;
2065 }
2066
2067 return VINF_SUCCESS;
2068}
2069
2070
2071/**
2072 * Does per-VM VT-x termination.
2073 *
2074 * @returns VBox status code.
2075 * @param pVM Pointer to the VM.
2076 */
2077VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2078{
2079 LogFlowFunc(("pVM=%p\n", pVM));
2080
2081#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2082 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2083 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2084#endif
2085 hmR0VmxStructsFree(pVM);
2086 return VINF_SUCCESS;
2087}
2088
2089
2090/**
2091 * Sets up the VM for execution under VT-x.
2092 * This function is only called once per-VM during initialization.
2093 *
2094 * @returns VBox status code.
2095 * @param pVM Pointer to the VM.
2096 */
2097VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2098{
2099 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2101
2102 LogFlowFunc(("pVM=%p\n", pVM));
2103
2104 /*
2105 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2106 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2107 */
2108 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2109 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2110 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2111 || !pVM->hm.s.vmx.pRealModeTSS))
2112 {
2113 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2114 return VERR_INTERNAL_ERROR;
2115 }
2116
2117#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2118 /*
2119 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2120 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2121 */
2122 if ( pVM->hm.s.fAllow64BitGuests
2123 && !HMVMX_IS_64BIT_HOST_MODE())
2124 {
2125 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2126 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2127 }
2128#endif
2129
2130 /* Initialize these always, see hmR3InitFinalizeR0().*/
2131 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2132 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2133
2134 /* Setup the tagged-TLB flush handlers. */
2135 int rc = hmR0VmxSetupTaggedTlb(pVM);
2136 if (RT_FAILURE(rc))
2137 {
2138 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2139 return rc;
2140 }
2141
2142 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2143 {
2144 PVMCPU pVCpu = &pVM->aCpus[i];
2145 AssertPtr(pVCpu);
2146 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2147
2148 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2149 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2150
2151 /* Set revision dword at the beginning of the VMCS structure. */
2152 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2153
2154 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2155 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2156 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2157 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2158
2159 /* Load this VMCS as the current VMCS. */
2160 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2161 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2162 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2163
2164 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2165 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2166 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2167
2168 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2169 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2170 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2171
2172 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2173 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2174 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2175
2176 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2177 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2178 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2179
2180 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2181 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2182 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2183
2184#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2185 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2186 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2187 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2188#endif
2189
2190 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2191 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2192 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2193 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2194
2195 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2196
2197 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2198 }
2199
2200 return VINF_SUCCESS;
2201}
2202
2203
2204/**
2205 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2206 * the VMCS.
2207 *
2208 * @returns VBox status code.
2209 * @param pVM Pointer to the VM.
2210 * @param pVCpu Pointer to the VMCPU.
2211 */
2212DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2213{
2214 RTCCUINTREG uReg = ASMGetCR0();
2215 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2216 AssertRCReturn(rc, rc);
2217
2218#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2219 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2220 if (HMVMX_IS_64BIT_HOST_MODE())
2221 {
2222 uint64_t uRegCR3 = HMR0Get64bitCR3();
2223 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2224 }
2225 else
2226#endif
2227 {
2228 uReg = ASMGetCR3();
2229 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2230 }
2231 AssertRCReturn(rc, rc);
2232
2233 uReg = ASMGetCR4();
2234 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2235 AssertRCReturn(rc, rc);
2236 return rc;
2237}
2238
2239
2240/**
2241 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2242 * the host-state area in the VMCS.
2243 *
2244 * @returns VBox status code.
2245 * @param pVM Pointer to the VM.
2246 * @param pVCpu Pointer to the VMCPU.
2247 */
2248DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2249{
2250 int rc = VERR_INTERNAL_ERROR_5;
2251
2252 /*
2253 * Host DS, ES, FS and GS segment registers.
2254 */
2255#if HC_ARCH_BITS == 64
2256 RTSEL uSelDS = ASMGetDS();
2257 RTSEL uSelES = ASMGetES();
2258 RTSEL uSelFS = ASMGetFS();
2259 RTSEL uSelGS = ASMGetGS();
2260#else
2261 RTSEL uSelDS = 0;
2262 RTSEL uSelES = 0;
2263 RTSEL uSelFS = 0;
2264 RTSEL uSelGS = 0;
2265#endif
2266
2267 /* Recalculate which host-state bits need to be manually restored. */
2268 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2269
2270 /*
2271 * Host CS and SS segment registers.
2272 */
2273#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2274 RTSEL uSelCS;
2275 RTSEL uSelSS;
2276 if (HMVMX_IS_64BIT_HOST_MODE())
2277 {
2278 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2279 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2280 }
2281 else
2282 {
2283 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2284 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2285 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2286 }
2287#else
2288 RTSEL uSelCS = ASMGetCS();
2289 RTSEL uSelSS = ASMGetSS();
2290#endif
2291
2292 /*
2293 * Host TR segment register.
2294 */
2295 RTSEL uSelTR = ASMGetTR();
2296
2297#if HC_ARCH_BITS == 64
2298 /*
2299 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2300 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2301 */
2302 if (uSelDS & (X86_SEL_RPL | X86_SEL_LDT))
2303 {
2304 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_DS;
2305 pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2306 uSelDS = 0;
2307 }
2308 if (uSelES & (X86_SEL_RPL | X86_SEL_LDT))
2309 {
2310 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_ES;
2311 pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2312 uSelES = 0;
2313 }
2314 if (uSelFS & (X86_SEL_RPL | X86_SEL_LDT))
2315 {
2316 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_FS;
2317 pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2318 uSelFS = 0;
2319 }
2320 if (uSelGS & (X86_SEL_RPL | X86_SEL_LDT))
2321 {
2322 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_GS;
2323 pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2324 uSelGS = 0;
2325 }
2326#endif
2327
2328 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2329 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2330 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2331 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2332 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2333 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2334 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2335 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2336 Assert(uSelCS);
2337 Assert(uSelTR);
2338
2339 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2340#if 0
2341 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2342 Assert(uSelSS != 0);
2343#endif
2344
2345 /* Write these host selector fields into the host-state area in the VMCS. */
2346 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2347 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2348#if HC_ARCH_BITS == 64
2349 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2350 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2351 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2352 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2353#endif
2354 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2355
2356 /*
2357 * Host GDTR and IDTR.
2358 */
2359 RTGDTR Gdtr;
2360 RT_ZERO(Gdtr);
2361#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2362 if (HMVMX_IS_64BIT_HOST_MODE())
2363 {
2364 X86XDTR64 Gdtr64;
2365 X86XDTR64 Idtr64;
2366 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2367 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2368 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2369
2370 Gdtr.cbGdt = Gdtr64.cb;
2371 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2372 }
2373 else
2374#endif
2375 {
2376 RTIDTR Idtr;
2377 ASMGetGDTR(&Gdtr);
2378 ASMGetIDTR(&Idtr);
2379 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2380 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2381
2382#if HC_ARCH_BITS == 64
2383 /*
2384 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2385 * maximum limit (0xffff) on every VM-exit.
2386 */
2387 if (Gdtr.cbGdt != 0xffff)
2388 {
2389 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2390 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2391 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2392 }
2393
2394 /*
2395 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2396 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2397 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2398 */
2399 if (Idtr.cbIdt < 0x0fff)
2400 {
2401 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2402 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2403 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2404 }
2405#endif
2406 }
2407
2408 /*
2409 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2410 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2411 */
2412 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2413 {
2414 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2415 return VERR_VMX_INVALID_HOST_STATE;
2416 }
2417
2418 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2419#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2420 if (HMVMX_IS_64BIT_HOST_MODE())
2421 {
2422 /* We need the 64-bit TR base for hybrid darwin. */
2423 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2424 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2425 }
2426 else
2427#endif
2428 {
2429 uintptr_t uTRBase;
2430#if HC_ARCH_BITS == 64
2431 uTRBase = X86DESC64_BASE(pDesc);
2432
2433 /*
2434 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2435 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2436 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2437 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2438 *
2439 * [1] See Intel spec. 3.5 "System Descriptor Types".
2440 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2441 */
2442 Assert(pDesc->System.u4Type == 11);
2443 if ( pDesc->System.u16LimitLow != 0x67
2444 || pDesc->System.u4LimitHigh)
2445 {
2446 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2447 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2448
2449 /* Store the GDTR here as we need it while restoring TR. */
2450 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2451 }
2452#else
2453 uTRBase = X86DESC_BASE(pDesc);
2454#endif
2455 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2456 }
2457 AssertRCReturn(rc, rc);
2458
2459 /*
2460 * Host FS base and GS base.
2461 */
2462#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2463 if (HMVMX_IS_64BIT_HOST_MODE())
2464 {
2465 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2466 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2467 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2468 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2469
2470# if HC_ARCH_BITS == 64
2471 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2472 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2473 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2474 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2475 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2476# endif
2477 }
2478#endif
2479 return rc;
2480}
2481
2482
2483/**
2484 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2485 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2486 * the host after every successful VM exit.
2487 *
2488 * @returns VBox status code.
2489 * @param pVM Pointer to the VM.
2490 * @param pVCpu Pointer to the VMCPU.
2491 */
2492DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2493{
2494 AssertPtr(pVCpu);
2495 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2496
2497 int rc = VINF_SUCCESS;
2498#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2499 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2500 uint32_t cHostMsrs = 0;
2501 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2502
2503 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2504 {
2505 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2506
2507# if HC_ARCH_BITS == 64
2508 /* Paranoia. 64-bit code requires these bits to be set always. */
2509 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2510
2511 /*
2512 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2513 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2514 * some reason (e.g. allow transparent reads) we would activate the code below.
2515 */
2516# if 0
2517 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2518 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2519 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2520 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2521 if (CPUMIsGuestInLongMode(pVCpu))
2522 {
2523 uint64_t u64GuestEfer;
2524 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2525 AssertRC(rc);
2526
2527 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2528 {
2529 pHostMsr->u32Msr = MSR_K6_EFER;
2530 pHostMsr->u32Reserved = 0;
2531 pHostMsr->u64Value = u64HostEfer;
2532 pHostMsr++; cHostMsrs++;
2533 }
2534 }
2535# endif
2536# else /* HC_ARCH_BITS != 64 */
2537 pHostMsr->u32Msr = MSR_K6_EFER;
2538 pHostMsr->u32Reserved = 0;
2539# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2540 if (CPUMIsGuestInLongMode(pVCpu))
2541 {
2542 /* Must match the EFER value in our 64 bits switcher. */
2543 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2544 }
2545 else
2546# endif
2547 pHostMsr->u64Value = u64HostEfer;
2548 pHostMsr++; cHostMsrs++;
2549# endif /* HC_ARCH_BITS == 64 */
2550 }
2551
2552# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2553 if (HMVMX_IS_64BIT_HOST_MODE())
2554 {
2555 pHostMsr->u32Msr = MSR_K6_STAR;
2556 pHostMsr->u32Reserved = 0;
2557 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2558 pHostMsr++; cHostMsrs++;
2559 pHostMsr->u32Msr = MSR_K8_LSTAR;
2560 pHostMsr->u32Reserved = 0;
2561 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2562 pHostMsr++; cHostMsrs++;
2563 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2564 pHostMsr->u32Reserved = 0;
2565 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2566 pHostMsr++; cHostMsrs++;
2567 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2568 pHostMsr->u32Reserved = 0;
2569 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2570 pHostMsr++; cHostMsrs++;
2571 }
2572# endif
2573
2574 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2575 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2576 {
2577 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2578 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2579 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2580 }
2581
2582 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2583#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2584
2585 /*
2586 * Host Sysenter MSRs.
2587 */
2588 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2589 AssertRCReturn(rc, rc);
2590#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2591 if (HMVMX_IS_64BIT_HOST_MODE())
2592 {
2593 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2594 AssertRCReturn(rc, rc);
2595 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2596 }
2597 else
2598 {
2599 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2600 AssertRCReturn(rc, rc);
2601 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2602 }
2603#elif HC_ARCH_BITS == 32
2604 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2605 AssertRCReturn(rc, rc);
2606 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2607#else
2608 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2609 AssertRCReturn(rc, rc);
2610 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2611#endif
2612 AssertRCReturn(rc, rc);
2613
2614 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2615 * hmR0VmxSetupExitCtls() !! */
2616 return rc;
2617}
2618
2619
2620/**
2621 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2622 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2623 * controls".
2624 *
2625 * @returns VBox status code.
2626 * @param pVCpu Pointer to the VMCPU.
2627 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2628 * out-of-sync. Make sure to update the required fields
2629 * before using them.
2630 *
2631 * @remarks No-long-jump zone!!!
2632 */
2633DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2634{
2635 int rc = VINF_SUCCESS;
2636 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
2637 {
2638 PVM pVM = pVCpu->CTX_SUFF(pVM);
2639 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2640 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2641
2642 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2643 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2644
2645 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2646 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2647 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2648 else
2649 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2650
2651 /*
2652 * The following should -not- be set (since we're not in SMM mode):
2653 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2654 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2655 */
2656
2657 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2658 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2659 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2660
2661 if ((val & zap) != val)
2662 {
2663 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2664 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2665 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2666 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2667 }
2668
2669 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2670 AssertRCReturn(rc, rc);
2671
2672 /* Update VCPU with the currently set VM-exit controls. */
2673 pVCpu->hm.s.vmx.u32EntryCtls = val;
2674 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
2675 }
2676 return rc;
2677}
2678
2679
2680/**
2681 * Sets up the VM-exit controls in the VMCS.
2682 *
2683 * @returns VBox status code.
2684 * @param pVM Pointer to the VM.
2685 * @param pVCpu Pointer to the VMCPU.
2686 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2687 * out-of-sync. Make sure to update the required fields
2688 * before using them.
2689 *
2690 * @remarks requires EFER.
2691 */
2692DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2693{
2694 int rc = VINF_SUCCESS;
2695 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
2696 {
2697 PVM pVM = pVCpu->CTX_SUFF(pVM);
2698 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2699 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2700
2701 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2702 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2703
2704 /*
2705 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2706 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2707 */
2708#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2709 if (HMVMX_IS_64BIT_HOST_MODE())
2710 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2711 else
2712 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2713#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2714 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2715 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2716 else
2717 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2718#endif
2719
2720 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2721 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2722
2723 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2724 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2725 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2726 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2727 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2728
2729 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2730 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2731
2732 if ((val & zap) != val)
2733 {
2734 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2735 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2736 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2737 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2738 }
2739
2740 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2741 AssertRCReturn(rc, rc);
2742
2743 /* Update VCPU with the currently set VM-exit controls. */
2744 pVCpu->hm.s.vmx.u32ExitCtls = val;
2745 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
2746 }
2747 return rc;
2748}
2749
2750
2751/**
2752 * Loads the guest APIC and related state.
2753 *
2754 * @returns VBox status code.
2755 * @param pVM Pointer to the VM.
2756 * @param pVCpu Pointer to the VMCPU.
2757 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2758 * out-of-sync. Make sure to update the required fields
2759 * before using them.
2760 */
2761DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2762{
2763 int rc = VINF_SUCCESS;
2764 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
2765 {
2766 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2767 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2768 {
2769 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2770
2771 bool fPendingIntr = false;
2772 uint8_t u8Tpr = 0;
2773 uint8_t u8PendingIntr = 0;
2774 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2775 AssertRCReturn(rc, rc);
2776
2777 /*
2778 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2779 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2780 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2781 * the interrupt when we VM-exit for other reasons.
2782 */
2783 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2784 uint32_t u32TprThreshold = 0;
2785 if (fPendingIntr)
2786 {
2787 /* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2788 const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2789 const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2790 if (u8PendingPriority <= u8TprPriority)
2791 u32TprThreshold = u8PendingPriority;
2792 else
2793 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2794 }
2795 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2796
2797 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2798 AssertRCReturn(rc, rc);
2799 }
2800
2801 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
2802 }
2803 return rc;
2804}
2805
2806
2807/**
2808 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2809 *
2810 * @returns Guest's interruptibility-state.
2811 * @param pVCpu Pointer to the VMCPU.
2812 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2813 * out-of-sync. Make sure to update the required fields
2814 * before using them.
2815 *
2816 * @remarks No-long-jump zone!!!
2817 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2818 */
2819DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2820{
2821 /*
2822 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2823 * inhibit interrupts or clear any existing interrupt-inhibition.
2824 */
2825 uint32_t uIntrState = 0;
2826 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2827 {
2828 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2829 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
2830 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2831 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2832 {
2833 /*
2834 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2835 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2836 */
2837 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2838 }
2839 else if (pMixedCtx->eflags.Bits.u1IF)
2840 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2841 else
2842 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2843 }
2844 return uIntrState;
2845}
2846
2847
2848/**
2849 * Loads the guest's interruptibility-state into the guest-state area in the
2850 * VMCS.
2851 *
2852 * @returns VBox status code.
2853 * @param pVCpu Pointer to the VMCPU.
2854 * @param uIntrState The interruptibility-state to set.
2855 */
2856static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2857{
2858 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2859 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2860 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2861 AssertRCReturn(rc, rc);
2862 return rc;
2863}
2864
2865
2866/**
2867 * Loads the guest's RIP into the guest-state area in the VMCS.
2868 *
2869 * @returns VBox status code.
2870 * @param pVCpu Pointer to the VMCPU.
2871 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2872 * out-of-sync. Make sure to update the required fields
2873 * before using them.
2874 *
2875 * @remarks No-long-jump zone!!!
2876 */
2877static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2878{
2879 int rc = VINF_SUCCESS;
2880 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
2881 {
2882 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2883 AssertRCReturn(rc, rc);
2884
2885 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
2886 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, VMCPU_HMCF_VALUE(pVCpu)));
2887 }
2888 return rc;
2889}
2890
2891
2892/**
2893 * Loads the guest's RSP into the guest-state area in the VMCS.
2894 *
2895 * @returns VBox status code.
2896 * @param pVCpu Pointer to the VMCPU.
2897 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2898 * out-of-sync. Make sure to update the required fields
2899 * before using them.
2900 *
2901 * @remarks No-long-jump zone!!!
2902 */
2903static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2904{
2905 int rc = VINF_SUCCESS;
2906 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
2907 {
2908 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2909 AssertRCReturn(rc, rc);
2910
2911 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
2912 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2913 }
2914 return rc;
2915}
2916
2917
2918/**
2919 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
2920 *
2921 * @returns VBox status code.
2922 * @param pVCpu Pointer to the VMCPU.
2923 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2924 * out-of-sync. Make sure to update the required fields
2925 * before using them.
2926 *
2927 * @remarks No-long-jump zone!!!
2928 */
2929static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2930{
2931 int rc = VINF_SUCCESS;
2932 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
2933 {
2934 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2935 Let us assert it as such and use 32-bit VMWRITE. */
2936 Assert(!(pMixedCtx->rflags.u64 >> 32));
2937 X86EFLAGS Eflags = pMixedCtx->eflags;
2938 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2939 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2940
2941 /*
2942 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2943 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2944 */
2945 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2946 {
2947 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2948 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2949 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2950 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2951 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2952 }
2953
2954 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2955 AssertRCReturn(rc, rc);
2956
2957 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
2958 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2959 }
2960 return rc;
2961}
2962
2963
2964/**
2965 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2966 *
2967 * @returns VBox status code.
2968 * @param pVCpu Pointer to the VMCPU.
2969 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2970 * out-of-sync. Make sure to update the required fields
2971 * before using them.
2972 *
2973 * @remarks No-long-jump zone!!!
2974 */
2975DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2976{
2977 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2978 AssertRCReturn(rc, rc);
2979 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2980 AssertRCReturn(rc, rc);
2981 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2982 AssertRCReturn(rc, rc);
2983 return rc;
2984}
2985
2986
2987/**
2988 * Loads the guest CR0 control register into the guest-state area in the VMCS.
2989 * CR0 is partially shared with the host and we have to consider the FPU bits.
2990 *
2991 * @returns VBox status code.
2992 * @param pVM Pointer to the VM.
2993 * @param pVCpu Pointer to the VMCPU.
2994 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2995 * out-of-sync. Make sure to update the required fields
2996 * before using them.
2997 *
2998 * @remarks No-long-jump zone!!!
2999 */
3000static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3001{
3002 /*
3003 * Guest CR0.
3004 * Guest FPU.
3005 */
3006 int rc = VINF_SUCCESS;
3007 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3008 {
3009 Assert(!(pMixedCtx->cr0 >> 32));
3010 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3011 PVM pVM = pVCpu->CTX_SUFF(pVM);
3012
3013 /* The guest's view (read access) of its CR0 is unblemished. */
3014 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3015 AssertRCReturn(rc, rc);
3016 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3017
3018 /* Setup VT-x's view of the guest CR0. */
3019 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3020 if (pVM->hm.s.fNestedPaging)
3021 {
3022 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3023 {
3024 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3025 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3026 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3027 }
3028 else
3029 {
3030 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3031 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3032 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3033 }
3034
3035 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3036 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3037 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3038
3039 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3040 AssertRCReturn(rc, rc);
3041 }
3042 else
3043 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3044
3045 /*
3046 * Guest FPU bits.
3047 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3048 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3049 */
3050 u32GuestCR0 |= X86_CR0_NE;
3051 bool fInterceptNM = false;
3052 if (CPUMIsGuestFPUStateActive(pVCpu))
3053 {
3054 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3055 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3056 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3057 }
3058 else
3059 {
3060 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3061 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3062 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3063 }
3064
3065 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3066 bool fInterceptMF = false;
3067 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3068 fInterceptMF = true;
3069
3070 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3071 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3072 {
3073 Assert(PDMVmmDevHeapIsEnabled(pVM));
3074 Assert(pVM->hm.s.vmx.pRealModeTSS);
3075 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3076 fInterceptNM = true;
3077 fInterceptMF = true;
3078 }
3079 else
3080 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3081
3082 if (fInterceptNM)
3083 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3084 else
3085 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3086
3087 if (fInterceptMF)
3088 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3089 else
3090 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3091
3092 /* Additional intercepts for debugging, define these yourself explicitly. */
3093#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3094 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3095 | RT_BIT(X86_XCPT_BP)
3096 | RT_BIT(X86_XCPT_DB)
3097 | RT_BIT(X86_XCPT_DE)
3098 | RT_BIT(X86_XCPT_NM)
3099 | RT_BIT(X86_XCPT_UD)
3100 | RT_BIT(X86_XCPT_NP)
3101 | RT_BIT(X86_XCPT_SS)
3102 | RT_BIT(X86_XCPT_GP)
3103 | RT_BIT(X86_XCPT_PF)
3104 | RT_BIT(X86_XCPT_MF)
3105 ;
3106#elif defined(HMVMX_ALWAYS_TRAP_PF)
3107 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3108#endif
3109
3110 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3111
3112 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3113 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3114 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3115 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3116 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3117 else
3118 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3119
3120 u32GuestCR0 |= uSetCR0;
3121 u32GuestCR0 &= uZapCR0;
3122 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3123
3124 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3125 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3126 AssertRCReturn(rc, rc);
3127 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3128 AssertRCReturn(rc, rc);
3129 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3130
3131 /*
3132 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3133 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3134 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3135 */
3136 uint32_t u32CR0Mask = 0;
3137 u32CR0Mask = X86_CR0_PE
3138 | X86_CR0_NE
3139 | X86_CR0_WP
3140 | X86_CR0_PG
3141 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3142 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3143 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3144
3145 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3146 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3147 * and @bugref{6944}. */
3148#if 0
3149 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3150 u32CR0Mask &= ~X86_CR0_PE;
3151#endif
3152 if (pVM->hm.s.fNestedPaging)
3153 u32CR0Mask &= ~X86_CR0_WP;
3154
3155 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3156 if (fInterceptNM)
3157 {
3158 u32CR0Mask |= X86_CR0_TS
3159 | X86_CR0_MP;
3160 }
3161
3162 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3163 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3164 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3165 AssertRCReturn(rc, rc);
3166 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3167
3168 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3169 }
3170 return rc;
3171}
3172
3173
3174/**
3175 * Loads the guest control registers (CR3, CR4) into the guest-state area
3176 * in the VMCS.
3177 *
3178 * @returns VBox status code.
3179 * @param pVM Pointer to the VM.
3180 * @param pVCpu Pointer to the VMCPU.
3181 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3182 * out-of-sync. Make sure to update the required fields
3183 * before using them.
3184 *
3185 * @remarks No-long-jump zone!!!
3186 */
3187static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3188{
3189 int rc = VINF_SUCCESS;
3190 PVM pVM = pVCpu->CTX_SUFF(pVM);
3191
3192 /*
3193 * Guest CR2.
3194 * It's always loaded in the assembler code. Nothing to do here.
3195 */
3196
3197 /*
3198 * Guest CR3.
3199 */
3200 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3201 {
3202 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3203 if (pVM->hm.s.fNestedPaging)
3204 {
3205 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3206
3207 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3208 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3209 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3210 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3211
3212 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3213 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3214 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3215
3216 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3217 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3218 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3219 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3220
3221 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3222 AssertRCReturn(rc, rc);
3223 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3224
3225 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3226 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3227 {
3228 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3229 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3230 {
3231 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3232 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3233 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3234 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3235 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3236 }
3237
3238 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3239 have Unrestricted Execution to handle the guest when it's not using paging. */
3240 GCPhysGuestCR3 = pMixedCtx->cr3;
3241 }
3242 else
3243 {
3244 /*
3245 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3246 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3247 * EPT takes care of translating it to host-physical addresses.
3248 */
3249 RTGCPHYS GCPhys;
3250 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3251 Assert(PDMVmmDevHeapIsEnabled(pVM));
3252
3253 /* We obtain it here every time as the guest could have relocated this PCI region. */
3254 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3255 AssertRCReturn(rc, rc);
3256
3257 GCPhysGuestCR3 = GCPhys;
3258 }
3259
3260 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3261 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3262 }
3263 else
3264 {
3265 /* Non-nested paging case, just use the hypervisor's CR3. */
3266 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3267
3268 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3269 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3270 }
3271 AssertRCReturn(rc, rc);
3272
3273 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3274 }
3275
3276 /*
3277 * Guest CR4.
3278 */
3279 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3280 {
3281 Assert(!(pMixedCtx->cr4 >> 32));
3282 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3283
3284 /* The guest's view of its CR4 is unblemished. */
3285 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3286 AssertRCReturn(rc, rc);
3287 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3288
3289 /* Setup VT-x's view of the guest CR4. */
3290 /*
3291 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3292 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3293 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3294 */
3295 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3296 {
3297 Assert(pVM->hm.s.vmx.pRealModeTSS);
3298 Assert(PDMVmmDevHeapIsEnabled(pVM));
3299 u32GuestCR4 &= ~X86_CR4_VME;
3300 }
3301
3302 if (pVM->hm.s.fNestedPaging)
3303 {
3304 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3305 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3306 {
3307 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3308 u32GuestCR4 |= X86_CR4_PSE;
3309 /* Our identity mapping is a 32 bits page directory. */
3310 u32GuestCR4 &= ~X86_CR4_PAE;
3311 }
3312 /* else use guest CR4.*/
3313 }
3314 else
3315 {
3316 /*
3317 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3318 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3319 */
3320 switch (pVCpu->hm.s.enmShadowMode)
3321 {
3322 case PGMMODE_REAL: /* Real-mode. */
3323 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3324 case PGMMODE_32_BIT: /* 32-bit paging. */
3325 {
3326 u32GuestCR4 &= ~X86_CR4_PAE;
3327 break;
3328 }
3329
3330 case PGMMODE_PAE: /* PAE paging. */
3331 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3332 {
3333 u32GuestCR4 |= X86_CR4_PAE;
3334 break;
3335 }
3336
3337 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3338 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3339#ifdef VBOX_ENABLE_64_BITS_GUESTS
3340 break;
3341#endif
3342 default:
3343 AssertFailed();
3344 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3345 }
3346 }
3347
3348 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3349 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3350 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3351 u32GuestCR4 |= uSetCR4;
3352 u32GuestCR4 &= uZapCR4;
3353
3354 /* Write VT-x's view of the guest CR4 into the VMCS. */
3355 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3356 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3357 AssertRCReturn(rc, rc);
3358
3359 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3360 uint32_t u32CR4Mask = 0;
3361 u32CR4Mask = X86_CR4_VME
3362 | X86_CR4_PAE
3363 | X86_CR4_PGE
3364 | X86_CR4_PSE
3365 | X86_CR4_VMXE;
3366 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3367 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3368 AssertRCReturn(rc, rc);
3369
3370 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
3371 }
3372 return rc;
3373}
3374
3375
3376/**
3377 * Loads the guest debug registers into the guest-state area in the VMCS.
3378 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3379 *
3380 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3381 *
3382 * @returns VBox status code.
3383 * @param pVCpu Pointer to the VMCPU.
3384 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3385 * out-of-sync. Make sure to update the required fields
3386 * before using them.
3387 *
3388 * @remarks No-long-jump zone!!!
3389 */
3390static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3391{
3392 if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
3393 return VINF_SUCCESS;
3394
3395#ifdef VBOX_STRICT
3396 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3397 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3398 {
3399 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3400 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3401 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3402 }
3403#endif
3404
3405 int rc;
3406 PVM pVM = pVCpu->CTX_SUFF(pVM);
3407 bool fInterceptDB = false;
3408 bool fInterceptMovDRx = false;
3409 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3410 {
3411 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3412 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3413 {
3414 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3415 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3416 AssertRCReturn(rc, rc);
3417 Assert(fInterceptDB == false);
3418 }
3419 else
3420 {
3421 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3422 pVCpu->hm.s.fClearTrapFlag = true;
3423 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3424 fInterceptDB = true;
3425 }
3426 }
3427
3428 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3429 {
3430 /*
3431 * Use the combined guest and host DRx values found in the hypervisor
3432 * register set because the debugger has breakpoints active or someone
3433 * is single stepping on the host side without a monitor trap flag.
3434 *
3435 * Note! DBGF expects a clean DR6 state before executing guest code.
3436 */
3437#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3438 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3439 && !CPUMIsHyperDebugStateActivePending(pVCpu))
3440 {
3441 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3442 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3443 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3444 }
3445 else
3446#endif
3447 if (!CPUMIsHyperDebugStateActive(pVCpu))
3448 {
3449 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3450 Assert(CPUMIsHyperDebugStateActive(pVCpu));
3451 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3452 }
3453
3454 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3455 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3456 AssertRCReturn(rc, rc);
3457
3458 pVCpu->hm.s.fUsingHyperDR7 = true;
3459 fInterceptDB = true;
3460 fInterceptMovDRx = true;
3461 }
3462 else
3463 {
3464 /*
3465 * If the guest has enabled debug registers, we need to load them prior to
3466 * executing guest code so they'll trigger at the right time.
3467 */
3468 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3469 {
3470#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3471 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3472 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3473 {
3474 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3475 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3476 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3477 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3478 }
3479 else
3480#endif
3481 if (CPUMIsGuestDebugStateActive(pVCpu))
3482 {
3483 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3484 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3485 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3486 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3487 }
3488 }
3489 /*
3490 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3491 * must intercept #DB in order to maintain a correct DR6 guest value.
3492 */
3493#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3494 else if ( ( CPUMIsGuestInLongModeEx(pMixedCtx)
3495 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3496 || !CPUMIsGuestDebugStateActive(pVCpu))
3497#else
3498 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3499#endif
3500 {
3501 fInterceptMovDRx = true;
3502 fInterceptDB = true;
3503 }
3504
3505 /* Update guest DR7. */
3506 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3507 AssertRCReturn(rc, rc);
3508
3509 pVCpu->hm.s.fUsingHyperDR7 = false;
3510 }
3511
3512 /*
3513 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3514 */
3515 if (fInterceptDB)
3516 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3517 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3518 {
3519#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3520 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3521#endif
3522 }
3523 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3524 AssertRCReturn(rc, rc);
3525
3526 /*
3527 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3528 */
3529 if (fInterceptMovDRx)
3530 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3531 else
3532 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3533 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3534 AssertRCReturn(rc, rc);
3535
3536 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
3537 return VINF_SUCCESS;
3538}
3539
3540
3541#ifdef VBOX_STRICT
3542/**
3543 * Strict function to validate segment registers.
3544 *
3545 * @remarks ASSUMES CR0 is up to date.
3546 */
3547static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3548{
3549 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3550 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3551 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3552 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3553 && ( !CPUMIsGuestInRealModeEx(pCtx)
3554 && !CPUMIsGuestInV86ModeEx(pCtx)))
3555 {
3556 /* Protected mode checks */
3557 /* CS */
3558 Assert(pCtx->cs.Attr.n.u1Present);
3559 Assert(!(pCtx->cs.Attr.u & 0xf00));
3560 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3561 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3562 || !(pCtx->cs.Attr.n.u1Granularity));
3563 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3564 || (pCtx->cs.Attr.n.u1Granularity));
3565 /* CS cannot be loaded with NULL in protected mode. */
3566 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3567 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3568 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3569 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3570 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3571 else
3572 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3573 /* SS */
3574 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3575 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3576 if ( !(pCtx->cr0 & X86_CR0_PE)
3577 || pCtx->cs.Attr.n.u4Type == 3)
3578 {
3579 Assert(!pCtx->ss.Attr.n.u2Dpl);
3580 }
3581 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3582 {
3583 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3584 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3585 Assert(pCtx->ss.Attr.n.u1Present);
3586 Assert(!(pCtx->ss.Attr.u & 0xf00));
3587 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3588 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3589 || !(pCtx->ss.Attr.n.u1Granularity));
3590 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3591 || (pCtx->ss.Attr.n.u1Granularity));
3592 }
3593 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3594 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3595 {
3596 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3597 Assert(pCtx->ds.Attr.n.u1Present);
3598 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3599 Assert(!(pCtx->ds.Attr.u & 0xf00));
3600 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3601 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3602 || !(pCtx->ds.Attr.n.u1Granularity));
3603 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3604 || (pCtx->ds.Attr.n.u1Granularity));
3605 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3606 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3607 }
3608 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3609 {
3610 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3611 Assert(pCtx->es.Attr.n.u1Present);
3612 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3613 Assert(!(pCtx->es.Attr.u & 0xf00));
3614 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3615 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3616 || !(pCtx->es.Attr.n.u1Granularity));
3617 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3618 || (pCtx->es.Attr.n.u1Granularity));
3619 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3620 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3621 }
3622 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3623 {
3624 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3625 Assert(pCtx->fs.Attr.n.u1Present);
3626 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3627 Assert(!(pCtx->fs.Attr.u & 0xf00));
3628 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3629 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3630 || !(pCtx->fs.Attr.n.u1Granularity));
3631 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3632 || (pCtx->fs.Attr.n.u1Granularity));
3633 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3634 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3635 }
3636 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3637 {
3638 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3639 Assert(pCtx->gs.Attr.n.u1Present);
3640 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3641 Assert(!(pCtx->gs.Attr.u & 0xf00));
3642 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3643 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3644 || !(pCtx->gs.Attr.n.u1Granularity));
3645 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3646 || (pCtx->gs.Attr.n.u1Granularity));
3647 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3648 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3649 }
3650 /* 64-bit capable CPUs. */
3651# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3652 Assert(!(pCtx->cs.u64Base >> 32));
3653 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3654 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3655 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3656# endif
3657 }
3658 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3659 || ( CPUMIsGuestInRealModeEx(pCtx)
3660 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3661 {
3662 /* Real and v86 mode checks. */
3663 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3664 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3665 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3666 {
3667 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3668 }
3669 else
3670 {
3671 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3672 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3673 }
3674
3675 /* CS */
3676 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3677 Assert(pCtx->cs.u32Limit == 0xffff);
3678 Assert(u32CSAttr == 0xf3);
3679 /* SS */
3680 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3681 Assert(pCtx->ss.u32Limit == 0xffff);
3682 Assert(u32SSAttr == 0xf3);
3683 /* DS */
3684 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3685 Assert(pCtx->ds.u32Limit == 0xffff);
3686 Assert(u32DSAttr == 0xf3);
3687 /* ES */
3688 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3689 Assert(pCtx->es.u32Limit == 0xffff);
3690 Assert(u32ESAttr == 0xf3);
3691 /* FS */
3692 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3693 Assert(pCtx->fs.u32Limit == 0xffff);
3694 Assert(u32FSAttr == 0xf3);
3695 /* GS */
3696 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3697 Assert(pCtx->gs.u32Limit == 0xffff);
3698 Assert(u32GSAttr == 0xf3);
3699 /* 64-bit capable CPUs. */
3700# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3701 Assert(!(pCtx->cs.u64Base >> 32));
3702 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3703 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3704 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3705# endif
3706 }
3707}
3708#endif /* VBOX_STRICT */
3709
3710
3711/**
3712 * Writes a guest segment register into the guest-state area in the VMCS.
3713 *
3714 * @returns VBox status code.
3715 * @param pVCpu Pointer to the VMCPU.
3716 * @param idxSel Index of the selector in the VMCS.
3717 * @param idxLimit Index of the segment limit in the VMCS.
3718 * @param idxBase Index of the segment base in the VMCS.
3719 * @param idxAccess Index of the access rights of the segment in the VMCS.
3720 * @param pSelReg Pointer to the segment selector.
3721 * @param pCtx Pointer to the guest-CPU context.
3722 *
3723 * @remarks No-long-jump zone!!!
3724 */
3725static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3726 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3727{
3728 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3729 AssertRCReturn(rc, rc);
3730 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3731 AssertRCReturn(rc, rc);
3732 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3733 AssertRCReturn(rc, rc);
3734
3735 uint32_t u32Access = pSelReg->Attr.u;
3736 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3737 {
3738 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3739 u32Access = 0xf3;
3740 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3741 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3742 }
3743 else
3744 {
3745 /*
3746 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3747 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3748 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3749 * loaded in protected-mode have their attribute as 0.
3750 */
3751 if (!u32Access)
3752 u32Access = X86DESCATTR_UNUSABLE;
3753 }
3754
3755 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3756 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3757 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3758
3759 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3760 AssertRCReturn(rc, rc);
3761 return rc;
3762}
3763
3764
3765/**
3766 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3767 * into the guest-state area in the VMCS.
3768 *
3769 * @returns VBox status code.
3770 * @param pVM Pointer to the VM.
3771 * @param pVCPU Pointer to the VMCPU.
3772 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3773 * out-of-sync. Make sure to update the required fields
3774 * before using them.
3775 *
3776 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
3777 * @remarks No-long-jump zone!!!
3778 */
3779static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3780{
3781 int rc = VERR_INTERNAL_ERROR_5;
3782 PVM pVM = pVCpu->CTX_SUFF(pVM);
3783
3784 /*
3785 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
3786 */
3787 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
3788 {
3789 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3790 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3791 {
3792 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3793 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3794 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3795 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3796 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3797 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3798 }
3799
3800#ifdef VBOX_WITH_REM
3801 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3802 {
3803 Assert(pVM->hm.s.vmx.pRealModeTSS);
3804 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3805 if ( pVCpu->hm.s.vmx.fWasInRealMode
3806 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3807 {
3808 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3809 in real-mode (e.g. OpenBSD 4.0) */
3810 REMFlushTBs(pVM);
3811 Log4(("Load: Switch to protected mode detected!\n"));
3812 pVCpu->hm.s.vmx.fWasInRealMode = false;
3813 }
3814 }
3815#endif
3816 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3817 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3818 AssertRCReturn(rc, rc);
3819 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3820 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3821 AssertRCReturn(rc, rc);
3822 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3823 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3824 AssertRCReturn(rc, rc);
3825 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3826 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3827 AssertRCReturn(rc, rc);
3828 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3829 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3830 AssertRCReturn(rc, rc);
3831 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3832 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3833 AssertRCReturn(rc, rc);
3834
3835#ifdef VBOX_STRICT
3836 /* Validate. */
3837 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3838#endif
3839
3840 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
3841 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3842 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3843 }
3844
3845 /*
3846 * Guest TR.
3847 */
3848 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
3849 {
3850 /*
3851 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3852 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3853 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3854 */
3855 uint16_t u16Sel = 0;
3856 uint32_t u32Limit = 0;
3857 uint64_t u64Base = 0;
3858 uint32_t u32AccessRights = 0;
3859
3860 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3861 {
3862 u16Sel = pMixedCtx->tr.Sel;
3863 u32Limit = pMixedCtx->tr.u32Limit;
3864 u64Base = pMixedCtx->tr.u64Base;
3865 u32AccessRights = pMixedCtx->tr.Attr.u;
3866 }
3867 else
3868 {
3869 Assert(pVM->hm.s.vmx.pRealModeTSS);
3870 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3871
3872 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3873 RTGCPHYS GCPhys;
3874 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3875 AssertRCReturn(rc, rc);
3876
3877 X86DESCATTR DescAttr;
3878 DescAttr.u = 0;
3879 DescAttr.n.u1Present = 1;
3880 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3881
3882 u16Sel = 0;
3883 u32Limit = HM_VTX_TSS_SIZE;
3884 u64Base = GCPhys; /* in real-mode phys = virt. */
3885 u32AccessRights = DescAttr.u;
3886 }
3887
3888 /* Validate. */
3889 Assert(!(u16Sel & RT_BIT(2)));
3890 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3891 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3892 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3893 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3894 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3895 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3896 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3897 Assert( (u32Limit & 0xfff) == 0xfff
3898 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3899 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3900 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3901
3902 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3903 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3904 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3905 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3906
3907 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
3908 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3909 }
3910
3911 /*
3912 * Guest GDTR.
3913 */
3914 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
3915 {
3916 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3917 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3918
3919 /* Validate. */
3920 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3921
3922 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
3923 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3924 }
3925
3926 /*
3927 * Guest LDTR.
3928 */
3929 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
3930 {
3931 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3932 uint32_t u32Access = 0;
3933 if (!pMixedCtx->ldtr.Attr.u)
3934 u32Access = X86DESCATTR_UNUSABLE;
3935 else
3936 u32Access = pMixedCtx->ldtr.Attr.u;
3937
3938 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3939 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3940 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3941 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3942
3943 /* Validate. */
3944 if (!(u32Access & X86DESCATTR_UNUSABLE))
3945 {
3946 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3947 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3948 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3949 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3950 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3951 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3952 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3953 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3954 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3955 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3956 }
3957
3958 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
3959 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3960 }
3961
3962 /*
3963 * Guest IDTR.
3964 */
3965 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
3966 {
3967 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3968 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3969
3970 /* Validate. */
3971 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3972
3973 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
3974 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3975 }
3976
3977 return VINF_SUCCESS;
3978}
3979
3980
3981/**
3982 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3983 * areas. These MSRs will automatically be loaded to the host CPU on every
3984 * successful VM entry and stored from the host CPU on every successful VM exit.
3985 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
3986 *
3987 * @returns VBox status code.
3988 * @param pVCpu Pointer to the VMCPU.
3989 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3990 * out-of-sync. Make sure to update the required fields
3991 * before using them.
3992 *
3993 * @remarks No-long-jump zone!!!
3994 */
3995static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3996{
3997 AssertPtr(pVCpu);
3998 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3999
4000 /*
4001 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
4002 */
4003 int rc = VINF_SUCCESS;
4004 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4005 {
4006#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
4007 PVM pVM = pVCpu->CTX_SUFF(pVM);
4008 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4009 uint32_t cGuestMsrs = 0;
4010
4011 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
4012 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
4013 * when the guest really is in 64-bit mode. */
4014 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
4015 if (fSupportsLongMode)
4016 {
4017 pGuestMsr->u32Msr = MSR_K8_LSTAR;
4018 pGuestMsr->u32Reserved = 0;
4019 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
4020 pGuestMsr++; cGuestMsrs++;
4021 pGuestMsr->u32Msr = MSR_K6_STAR;
4022 pGuestMsr->u32Reserved = 0;
4023 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
4024 pGuestMsr++; cGuestMsrs++;
4025 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
4026 pGuestMsr->u32Reserved = 0;
4027 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
4028 pGuestMsr++; cGuestMsrs++;
4029 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4030 pGuestMsr->u32Reserved = 0;
4031 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4032 pGuestMsr++; cGuestMsrs++;
4033 }
4034
4035 /*
4036 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4037 * load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
4038 */
4039 if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
4040 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
4041 {
4042 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4043 pGuestMsr->u32Reserved = 0;
4044 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4045 AssertRCReturn(rc, rc);
4046 pGuestMsr++; cGuestMsrs++;
4047 }
4048
4049 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4050 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4051 {
4052 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4053 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4054 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4055 }
4056
4057 /* Update the VCPU's copy of the guest MSR count. */
4058 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4059 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4060 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4061#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4062
4063 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4064 }
4065
4066 /*
4067 * Guest Sysenter MSRs.
4068 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4069 * VM-exits on WRMSRs for these MSRs.
4070 */
4071 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4072 {
4073 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4074 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4075 }
4076
4077 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4078 {
4079 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4080 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4081 }
4082
4083 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4084 {
4085 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4086 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4087 }
4088
4089 return rc;
4090}
4091
4092
4093/**
4094 * Loads the guest activity state into the guest-state area in the VMCS.
4095 *
4096 * @returns VBox status code.
4097 * @param pVCpu Pointer to the VMCPU.
4098 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4099 * out-of-sync. Make sure to update the required fields
4100 * before using them.
4101 *
4102 * @remarks No-long-jump zone!!!
4103 */
4104static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4105{
4106 /** @todo See if we can make use of other states, e.g.
4107 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4108 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4109 {
4110 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4111 AssertRCReturn(rc, rc);
4112
4113 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4114 }
4115 return VINF_SUCCESS;
4116}
4117
4118
4119/**
4120 * Sets up the appropriate function to run guest code.
4121 *
4122 * @returns VBox status code.
4123 * @param pVCpu Pointer to the VMCPU.
4124 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4125 * out-of-sync. Make sure to update the required fields
4126 * before using them.
4127 *
4128 * @remarks No-long-jump zone!!!
4129 */
4130static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4131{
4132 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4133 {
4134#ifndef VBOX_ENABLE_64_BITS_GUESTS
4135 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4136#endif
4137 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4138#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4139 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4140 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4141 {
4142 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4143 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS);
4144 }
4145#else
4146 /* 64-bit host or hybrid host. */
4147 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4148#endif
4149 }
4150 else
4151 {
4152 /* Guest is not in long mode, use the 32-bit handler. */
4153#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4154 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4155 {
4156 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4157 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS);
4158 }
4159#else
4160 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4161#endif
4162 }
4163 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4164 return VINF_SUCCESS;
4165}
4166
4167
4168/**
4169 * Wrapper for running the guest code in VT-x.
4170 *
4171 * @returns VBox strict status code.
4172 * @param pVM Pointer to the VM.
4173 * @param pVCpu Pointer to the VMCPU.
4174 * @param pCtx Pointer to the guest-CPU context.
4175 *
4176 * @remarks No-long-jump zone!!!
4177 */
4178DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4179{
4180 /*
4181 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4182 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4183 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4184 */
4185 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4186 /** @todo Add stats for resume vs launch. */
4187#ifdef VBOX_WITH_KERNEL_USING_XMM
4188 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4189#else
4190 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4191#endif
4192}
4193
4194
4195/**
4196 * Reports world-switch error and dumps some useful debug info.
4197 *
4198 * @param pVM Pointer to the VM.
4199 * @param pVCpu Pointer to the VMCPU.
4200 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4201 * @param pCtx Pointer to the guest-CPU context.
4202 * @param pVmxTransient Pointer to the VMX transient structure (only
4203 * exitReason updated).
4204 */
4205static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4206{
4207 Assert(pVM);
4208 Assert(pVCpu);
4209 Assert(pCtx);
4210 Assert(pVmxTransient);
4211 HMVMX_ASSERT_PREEMPT_SAFE();
4212
4213 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4214 switch (rcVMRun)
4215 {
4216 case VERR_VMX_INVALID_VMXON_PTR:
4217 AssertFailed();
4218 break;
4219 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4220 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4221 {
4222 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4223 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4224 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4225 AssertRC(rc);
4226
4227 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4228 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4229 Cannot do it here as we may have been long preempted. */
4230
4231#ifdef VBOX_STRICT
4232 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4233 pVmxTransient->uExitReason));
4234 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4235 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4236 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4237 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4238 else
4239 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4240 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4241 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4242
4243 /* VMX control bits. */
4244 uint32_t u32Val;
4245 uint64_t u64Val;
4246 HMVMXHCUINTREG uHCReg;
4247 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4248 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4249 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4250 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4251 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4252 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4253 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4254 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4255 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4256 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4257 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4258 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4259 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4260 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4261 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4262 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4263 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4264 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4265 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4266 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4267 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4268 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4269 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4270 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4271 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4272 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4273 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4274 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4275 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4276 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4277 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4278 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4279 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4280 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4281 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4282 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4283 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4284 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4285 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4286 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4287 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4288 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4289
4290 /* Guest bits. */
4291 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4292 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4293 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4294 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4295 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4296 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4297 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4298 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4299
4300 /* Host bits. */
4301 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4302 Log4(("Host CR0 %#RHr\n", uHCReg));
4303 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4304 Log4(("Host CR3 %#RHr\n", uHCReg));
4305 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4306 Log4(("Host CR4 %#RHr\n", uHCReg));
4307
4308 RTGDTR HostGdtr;
4309 PCX86DESCHC pDesc;
4310 ASMGetGDTR(&HostGdtr);
4311 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4312 Log4(("Host CS %#08x\n", u32Val));
4313 if (u32Val < HostGdtr.cbGdt)
4314 {
4315 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4316 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4317 }
4318
4319 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4320 Log4(("Host DS %#08x\n", u32Val));
4321 if (u32Val < HostGdtr.cbGdt)
4322 {
4323 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4324 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4325 }
4326
4327 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4328 Log4(("Host ES %#08x\n", u32Val));
4329 if (u32Val < HostGdtr.cbGdt)
4330 {
4331 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4332 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4333 }
4334
4335 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4336 Log4(("Host FS %#08x\n", u32Val));
4337 if (u32Val < HostGdtr.cbGdt)
4338 {
4339 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4340 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4341 }
4342
4343 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4344 Log4(("Host GS %#08x\n", u32Val));
4345 if (u32Val < HostGdtr.cbGdt)
4346 {
4347 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4348 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4349 }
4350
4351 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4352 Log4(("Host SS %#08x\n", u32Val));
4353 if (u32Val < HostGdtr.cbGdt)
4354 {
4355 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4356 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4357 }
4358
4359 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4360 Log4(("Host TR %#08x\n", u32Val));
4361 if (u32Val < HostGdtr.cbGdt)
4362 {
4363 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4364 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4365 }
4366
4367 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4368 Log4(("Host TR Base %#RHv\n", uHCReg));
4369 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4370 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4371 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4372 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4373 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4374 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4375 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4376 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4377 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4378 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4379 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4380 Log4(("Host RSP %#RHv\n", uHCReg));
4381 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4382 Log4(("Host RIP %#RHv\n", uHCReg));
4383# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4384 if (HMVMX_IS_64BIT_HOST_MODE())
4385 {
4386 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4387 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4388 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4389 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4390 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4391 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4392 }
4393# endif
4394#endif /* VBOX_STRICT */
4395 break;
4396 }
4397
4398 default:
4399 /* Impossible */
4400 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4401 break;
4402 }
4403 NOREF(pVM);
4404}
4405
4406
4407#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4408#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4409# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4410#endif
4411#ifdef VBOX_STRICT
4412static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4413{
4414 switch (idxField)
4415 {
4416 case VMX_VMCS_GUEST_RIP:
4417 case VMX_VMCS_GUEST_RSP:
4418 case VMX_VMCS_GUEST_SYSENTER_EIP:
4419 case VMX_VMCS_GUEST_SYSENTER_ESP:
4420 case VMX_VMCS_GUEST_GDTR_BASE:
4421 case VMX_VMCS_GUEST_IDTR_BASE:
4422 case VMX_VMCS_GUEST_CS_BASE:
4423 case VMX_VMCS_GUEST_DS_BASE:
4424 case VMX_VMCS_GUEST_ES_BASE:
4425 case VMX_VMCS_GUEST_FS_BASE:
4426 case VMX_VMCS_GUEST_GS_BASE:
4427 case VMX_VMCS_GUEST_SS_BASE:
4428 case VMX_VMCS_GUEST_LDTR_BASE:
4429 case VMX_VMCS_GUEST_TR_BASE:
4430 case VMX_VMCS_GUEST_CR3:
4431 return true;
4432 }
4433 return false;
4434}
4435
4436static bool hmR0VmxIsValidReadField(uint32_t idxField)
4437{
4438 switch (idxField)
4439 {
4440 /* Read-only fields. */
4441 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4442 return true;
4443 }
4444 /* Remaining readable fields should also be writable. */
4445 return hmR0VmxIsValidWriteField(idxField);
4446}
4447#endif /* VBOX_STRICT */
4448
4449
4450/**
4451 * Executes the specified handler in 64-bit mode.
4452 *
4453 * @returns VBox status code.
4454 * @param pVM Pointer to the VM.
4455 * @param pVCpu Pointer to the VMCPU.
4456 * @param pCtx Pointer to the guest CPU context.
4457 * @param enmOp The operation to perform.
4458 * @param cbParam Number of parameters.
4459 * @param paParam Array of 32-bit parameters.
4460 */
4461VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4462 uint32_t *paParam)
4463{
4464 int rc, rc2;
4465 PHMGLOBALCPUINFO pCpu;
4466 RTHCPHYS HCPhysCpuPage;
4467 RTCCUINTREG uOldEflags;
4468
4469 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4470 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4471 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4472 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4473
4474#ifdef VBOX_STRICT
4475 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4476 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4477
4478 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4479 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4480#endif
4481
4482 /* Disable interrupts. */
4483 uOldEflags = ASMIntDisableFlags();
4484
4485#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4486 RTCPUID idHostCpu = RTMpCpuId();
4487 CPUMR0SetLApic(pVCpu, idHostCpu);
4488#endif
4489
4490 pCpu = HMR0GetCurrentCpu();
4491 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4492
4493 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4494 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4495
4496 /* Leave VMX Root Mode. */
4497 VMXDisable();
4498
4499 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4500
4501 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4502 CPUMSetHyperEIP(pVCpu, enmOp);
4503 for (int i = (int)cbParam - 1; i >= 0; i--)
4504 CPUMPushHyper(pVCpu, paParam[i]);
4505
4506 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4507
4508 /* Call the switcher. */
4509 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4510 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4511
4512 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4513 /* Make sure the VMX instructions don't cause #UD faults. */
4514 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4515
4516 /* Re-enter VMX Root Mode */
4517 rc2 = VMXEnable(HCPhysCpuPage);
4518 if (RT_FAILURE(rc2))
4519 {
4520 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4521 ASMSetFlags(uOldEflags);
4522 return rc2;
4523 }
4524
4525 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4526 AssertRC(rc2);
4527 Assert(!(ASMGetFlags() & X86_EFL_IF));
4528 ASMSetFlags(uOldEflags);
4529 return rc;
4530}
4531
4532
4533/**
4534 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4535 * supporting 64-bit guests.
4536 *
4537 * @returns VBox status code.
4538 * @param fResume Whether to VMLAUNCH or VMRESUME.
4539 * @param pCtx Pointer to the guest-CPU context.
4540 * @param pCache Pointer to the VMCS cache.
4541 * @param pVM Pointer to the VM.
4542 * @param pVCpu Pointer to the VMCPU.
4543 */
4544DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4545{
4546 uint32_t aParam[6];
4547 PHMGLOBALCPUINFO pCpu = NULL;
4548 RTHCPHYS HCPhysCpuPage = 0;
4549 int rc = VERR_INTERNAL_ERROR_5;
4550
4551 pCpu = HMR0GetCurrentCpu();
4552 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4553
4554#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4555 pCache->uPos = 1;
4556 pCache->interPD = PGMGetInterPaeCR3(pVM);
4557 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4558#endif
4559
4560#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4561 pCache->TestIn.HCPhysCpuPage = 0;
4562 pCache->TestIn.HCPhysVmcs = 0;
4563 pCache->TestIn.pCache = 0;
4564 pCache->TestOut.HCPhysVmcs = 0;
4565 pCache->TestOut.pCache = 0;
4566 pCache->TestOut.pCtx = 0;
4567 pCache->TestOut.eflags = 0;
4568#endif
4569
4570 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4571 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4572 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4573 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4574 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4575 aParam[5] = 0;
4576
4577#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4578 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4579 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4580#endif
4581 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4582
4583#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4584 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4585 Assert(pCtx->dr[4] == 10);
4586 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4587#endif
4588
4589#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
4590 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4591 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4592 pVCpu->hm.s.vmx.HCPhysVmcs));
4593 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4594 pCache->TestOut.HCPhysVmcs));
4595 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4596 pCache->TestOut.pCache));
4597 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4598 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4599 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4600 pCache->TestOut.pCtx));
4601 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4602#endif
4603 return rc;
4604}
4605
4606
4607/**
4608 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4609 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4610 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4611 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4612 *
4613 * @returns VBox status code.
4614 * @param pVM Pointer to the VM.
4615 * @param pVCpu Pointer to the VMCPU.
4616 */
4617static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4618{
4619#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4620{ \
4621 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4622 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4623 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4624 ++cReadFields; \
4625}
4626
4627 AssertPtr(pVM);
4628 AssertPtr(pVCpu);
4629 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4630 uint32_t cReadFields = 0;
4631
4632 /*
4633 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4634 * and serve to indicate exceptions to the rules.
4635 */
4636
4637 /* Guest-natural selector base fields. */
4638#if 0
4639 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4640 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4641 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4642#endif
4643 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4644 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4645 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4646 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4647 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4648 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4649 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4650 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4651 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4652 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4653 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4654 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4655#if 0
4656 /* Unused natural width guest-state fields. */
4657 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4658 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4659#endif
4660 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4661 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4662
4663 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4664#if 0
4665 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4666 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4667 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4668 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4669 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4670 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4671 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4672 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4673 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4674#endif
4675
4676 /* Natural width guest-state fields. */
4677 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4678#if 0
4679 /* Currently unused field. */
4680 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4681#endif
4682
4683 if (pVM->hm.s.fNestedPaging)
4684 {
4685 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4686 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4687 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4688 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4689 }
4690 else
4691 {
4692 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4693 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4694 }
4695
4696#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4697 return VINF_SUCCESS;
4698}
4699
4700
4701/**
4702 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4703 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4704 * darwin, running 64-bit guests).
4705 *
4706 * @returns VBox status code.
4707 * @param pVCpu Pointer to the VMCPU.
4708 * @param idxField The VMCS field encoding.
4709 * @param u64Val 16, 32 or 64 bits value.
4710 */
4711VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4712{
4713 int rc;
4714 switch (idxField)
4715 {
4716 /*
4717 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4718 */
4719 /* 64-bit Control fields. */
4720 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4721 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4722 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4723 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4724 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4725 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4726 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4727 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4728 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4729 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4730 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4731 case VMX_VMCS64_CTRL_EPTP_FULL:
4732 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4733 /* 64-bit Guest-state fields. */
4734 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4735 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4736 case VMX_VMCS64_GUEST_PAT_FULL:
4737 case VMX_VMCS64_GUEST_EFER_FULL:
4738 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4739 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4740 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4741 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4742 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4743 /* 64-bit Host-state fields. */
4744 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4745 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4746 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4747 {
4748 rc = VMXWriteVmcs32(idxField, u64Val);
4749 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4750 break;
4751 }
4752
4753 /*
4754 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4755 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4756 */
4757 /* Natural-width Guest-state fields. */
4758 case VMX_VMCS_GUEST_CR3:
4759 case VMX_VMCS_GUEST_ES_BASE:
4760 case VMX_VMCS_GUEST_CS_BASE:
4761 case VMX_VMCS_GUEST_SS_BASE:
4762 case VMX_VMCS_GUEST_DS_BASE:
4763 case VMX_VMCS_GUEST_FS_BASE:
4764 case VMX_VMCS_GUEST_GS_BASE:
4765 case VMX_VMCS_GUEST_LDTR_BASE:
4766 case VMX_VMCS_GUEST_TR_BASE:
4767 case VMX_VMCS_GUEST_GDTR_BASE:
4768 case VMX_VMCS_GUEST_IDTR_BASE:
4769 case VMX_VMCS_GUEST_RSP:
4770 case VMX_VMCS_GUEST_RIP:
4771 case VMX_VMCS_GUEST_SYSENTER_ESP:
4772 case VMX_VMCS_GUEST_SYSENTER_EIP:
4773 {
4774 if (!(u64Val >> 32))
4775 {
4776 /* If this field is 64-bit, VT-x will zero out the top bits. */
4777 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4778 }
4779 else
4780 {
4781 /* Assert that only the 32->64 switcher case should ever come here. */
4782 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4783 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4784 }
4785 break;
4786 }
4787
4788 default:
4789 {
4790 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4791 rc = VERR_INVALID_PARAMETER;
4792 break;
4793 }
4794 }
4795 AssertRCReturn(rc, rc);
4796 return rc;
4797}
4798
4799
4800/**
4801 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4802 * hosts (except darwin) for 64-bit guests.
4803 *
4804 * @param pVCpu Pointer to the VMCPU.
4805 * @param idxField The VMCS field encoding.
4806 * @param u64Val 16, 32 or 64 bits value.
4807 */
4808VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4809{
4810 AssertPtr(pVCpu);
4811 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4812
4813 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4814 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4815
4816 /* Make sure there are no duplicates. */
4817 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4818 {
4819 if (pCache->Write.aField[i] == idxField)
4820 {
4821 pCache->Write.aFieldVal[i] = u64Val;
4822 return VINF_SUCCESS;
4823 }
4824 }
4825
4826 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4827 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4828 pCache->Write.cValidEntries++;
4829 return VINF_SUCCESS;
4830}
4831
4832/* Enable later when the assembly code uses these as callbacks. */
4833#if 0
4834/*
4835 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4836 *
4837 * @param pVCpu Pointer to the VMCPU.
4838 * @param pCache Pointer to the VMCS cache.
4839 *
4840 * @remarks No-long-jump zone!!!
4841 */
4842VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4843{
4844 AssertPtr(pCache);
4845 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4846 {
4847 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4848 AssertRC(rc);
4849 }
4850 pCache->Write.cValidEntries = 0;
4851}
4852
4853
4854/**
4855 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
4856 *
4857 * @param pVCpu Pointer to the VMCPU.
4858 * @param pCache Pointer to the VMCS cache.
4859 *
4860 * @remarks No-long-jump zone!!!
4861 */
4862VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4863{
4864 AssertPtr(pCache);
4865 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4866 {
4867 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4868 AssertRC(rc);
4869 }
4870}
4871#endif
4872#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4873
4874
4875/**
4876 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4877 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4878 * timer.
4879 *
4880 * @returns VBox status code.
4881 * @param pVCpu Pointer to the VMCPU.
4882 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4883 * out-of-sync. Make sure to update the required fields
4884 * before using them.
4885 * @remarks No-long-jump zone!!!
4886 */
4887static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4888{
4889 int rc = VERR_INTERNAL_ERROR_5;
4890 bool fOffsettedTsc = false;
4891 PVM pVM = pVCpu->CTX_SUFF(pVM);
4892 if (pVM->hm.s.vmx.fUsePreemptTimer)
4893 {
4894 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4895
4896 /* Make sure the returned values have sane upper and lower boundaries. */
4897 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4898 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4899 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4900 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4901
4902 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4903 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4904 }
4905 else
4906 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4907
4908 if (fOffsettedTsc)
4909 {
4910 uint64_t u64CurTSC = ASMReadTSC();
4911 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4912 {
4913 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4914 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4915
4916 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4917 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4918 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4919 }
4920 else
4921 {
4922 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4923 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4924 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4925 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4926 }
4927 }
4928 else
4929 {
4930 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4931 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4932 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4933 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4934 }
4935}
4936
4937
4938/**
4939 * Determines if an exception is a contributory exception. Contributory
4940 * exceptions are ones which can cause double-faults. Page-fault is
4941 * intentionally not included here as it's a conditional contributory exception.
4942 *
4943 * @returns true if the exception is contributory, false otherwise.
4944 * @param uVector The exception vector.
4945 */
4946DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4947{
4948 switch (uVector)
4949 {
4950 case X86_XCPT_GP:
4951 case X86_XCPT_SS:
4952 case X86_XCPT_NP:
4953 case X86_XCPT_TS:
4954 case X86_XCPT_DE:
4955 return true;
4956 default:
4957 break;
4958 }
4959 return false;
4960}
4961
4962
4963/**
4964 * Sets an event as a pending event to be injected into the guest.
4965 *
4966 * @param pVCpu Pointer to the VMCPU.
4967 * @param u32IntrInfo The VM-entry interruption-information field.
4968 * @param cbInstr The VM-entry instruction length in bytes (for software
4969 * interrupts, exceptions and privileged software
4970 * exceptions).
4971 * @param u32ErrCode The VM-entry exception error code.
4972 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4973 * page-fault.
4974 *
4975 * @remarks Statistics counter assumes this is a guest event being injected or
4976 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4977 * always incremented.
4978 */
4979DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4980 RTGCUINTPTR GCPtrFaultAddress)
4981{
4982 Assert(!pVCpu->hm.s.Event.fPending);
4983 pVCpu->hm.s.Event.fPending = true;
4984 pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4985 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4986 pVCpu->hm.s.Event.cbInstr = cbInstr;
4987 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4988
4989 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4990}
4991
4992
4993/**
4994 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4995 *
4996 * @param pVCpu Pointer to the VMCPU.
4997 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4998 * out-of-sync. Make sure to update the required fields
4999 * before using them.
5000 */
5001DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5002{
5003 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5004 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5005 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5006 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5007}
5008
5009
5010/**
5011 * Handle a condition that occurred while delivering an event through the guest
5012 * IDT.
5013 *
5014 * @returns VBox status code (informational error codes included).
5015 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5016 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5017 * continue execution of the guest which will delivery the #DF.
5018 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5019 *
5020 * @param pVCpu Pointer to the VMCPU.
5021 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5022 * out-of-sync. Make sure to update the required fields
5023 * before using them.
5024 * @param pVmxTransient Pointer to the VMX transient structure.
5025 *
5026 * @remarks No-long-jump zone!!!
5027 */
5028static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5029{
5030 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5031 AssertRCReturn(rc, rc);
5032 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5033 {
5034 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
5035 AssertRCReturn(rc, rc);
5036
5037 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5038 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
5039 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5040
5041 typedef enum
5042 {
5043 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5044 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5045 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5046 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5047 } VMXREFLECTXCPT;
5048
5049 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5050 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5051 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
5052 {
5053 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5054 {
5055 enmReflect = VMXREFLECTXCPT_XCPT;
5056#ifdef VBOX_STRICT
5057 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5058 && uExitVector == X86_XCPT_PF)
5059 {
5060 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5061 }
5062#endif
5063 if ( uExitVector == X86_XCPT_PF
5064 && uIdtVector == X86_XCPT_PF)
5065 {
5066 pVmxTransient->fVectoringPF = true;
5067 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5068 }
5069 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5070 && hmR0VmxIsContributoryXcpt(uExitVector)
5071 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5072 || uIdtVector == X86_XCPT_PF))
5073 {
5074 enmReflect = VMXREFLECTXCPT_DF;
5075 }
5076 else if (uIdtVector == X86_XCPT_DF)
5077 enmReflect = VMXREFLECTXCPT_TF;
5078 }
5079 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5080 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5081 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5082 {
5083 /*
5084 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5085 * (whatever they are) as they reoccur when restarting the instruction.
5086 */
5087 enmReflect = VMXREFLECTXCPT_XCPT;
5088 }
5089 }
5090 else
5091 {
5092 /*
5093 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5094 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5095 * original exception to the guest after handling the VM-exit.
5096 */
5097 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5098 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5099 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5100 {
5101 enmReflect = VMXREFLECTXCPT_XCPT;
5102 }
5103 }
5104
5105 switch (enmReflect)
5106 {
5107 case VMXREFLECTXCPT_XCPT:
5108 {
5109 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5110 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5111 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5112
5113 uint32_t u32ErrCode = 0;
5114 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5115 {
5116 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5117 AssertRCReturn(rc, rc);
5118 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5119 }
5120
5121 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5122 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5123 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5124 rc = VINF_SUCCESS;
5125 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5126 pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
5127
5128 break;
5129 }
5130
5131 case VMXREFLECTXCPT_DF:
5132 {
5133 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5134 rc = VINF_HM_DOUBLE_FAULT;
5135 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5136 pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
5137
5138 break;
5139 }
5140
5141 case VMXREFLECTXCPT_TF:
5142 {
5143 rc = VINF_EM_RESET;
5144 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5145 uExitVector));
5146 break;
5147 }
5148
5149 default:
5150 Assert(rc == VINF_SUCCESS);
5151 break;
5152 }
5153 }
5154 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5155 return rc;
5156}
5157
5158
5159/**
5160 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5161 *
5162 * @returns VBox status code.
5163 * @param pVCpu Pointer to the VMCPU.
5164 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5165 * out-of-sync. Make sure to update the required fields
5166 * before using them.
5167 *
5168 * @remarks No-long-jump zone!!!
5169 */
5170static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5171{
5172 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5173 {
5174 uint32_t uVal = 0;
5175 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5176 AssertRCReturn(rc, rc);
5177
5178 uint32_t uShadow = 0;
5179 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5180 AssertRCReturn(rc, rc);
5181
5182 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5183 CPUMSetGuestCR0(pVCpu, uVal);
5184 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5185 }
5186 return VINF_SUCCESS;
5187}
5188
5189
5190/**
5191 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5192 *
5193 * @returns VBox status code.
5194 * @param pVCpu Pointer to the VMCPU.
5195 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5196 * out-of-sync. Make sure to update the required fields
5197 * before using them.
5198 *
5199 * @remarks No-long-jump zone!!!
5200 */
5201static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5202{
5203 int rc = VINF_SUCCESS;
5204 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5205 {
5206 uint32_t uVal = 0;
5207 uint32_t uShadow = 0;
5208 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5209 AssertRCReturn(rc, rc);
5210 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5211 AssertRCReturn(rc, rc);
5212
5213 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5214 CPUMSetGuestCR4(pVCpu, uVal);
5215 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5216 }
5217 return rc;
5218}
5219
5220
5221/**
5222 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5223 *
5224 * @returns VBox status code.
5225 * @param pVCpu Pointer to the VMCPU.
5226 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5227 * out-of-sync. Make sure to update the required fields
5228 * before using them.
5229 *
5230 * @remarks No-long-jump zone!!!
5231 */
5232static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5233{
5234 int rc = VINF_SUCCESS;
5235 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5236 {
5237 uint64_t u64Val = 0;
5238 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5239 AssertRCReturn(rc, rc);
5240
5241 pMixedCtx->rip = u64Val;
5242 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5243 }
5244 return rc;
5245}
5246
5247
5248/**
5249 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5250 *
5251 * @returns VBox status code.
5252 * @param pVCpu Pointer to the VMCPU.
5253 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5254 * out-of-sync. Make sure to update the required fields
5255 * before using them.
5256 *
5257 * @remarks No-long-jump zone!!!
5258 */
5259static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5260{
5261 int rc = VINF_SUCCESS;
5262 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5263 {
5264 uint64_t u64Val = 0;
5265 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5266 AssertRCReturn(rc, rc);
5267
5268 pMixedCtx->rsp = u64Val;
5269 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5270 }
5271 return rc;
5272}
5273
5274
5275/**
5276 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5277 *
5278 * @returns VBox status code.
5279 * @param pVCpu Pointer to the VMCPU.
5280 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5281 * out-of-sync. Make sure to update the required fields
5282 * before using them.
5283 *
5284 * @remarks No-long-jump zone!!!
5285 */
5286static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5287{
5288 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5289 {
5290 uint32_t uVal = 0;
5291 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5292 AssertRCReturn(rc, rc);
5293
5294 pMixedCtx->eflags.u32 = uVal;
5295 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5296 {
5297 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5298 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5299
5300 pMixedCtx->eflags.Bits.u1VM = 0;
5301 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5302 }
5303
5304 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5305 }
5306 return VINF_SUCCESS;
5307}
5308
5309
5310/**
5311 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5312 * guest-CPU context.
5313 */
5314DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5315{
5316 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5317 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5318 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5319 return rc;
5320}
5321
5322
5323/**
5324 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5325 * from the guest-state area in the VMCS.
5326 *
5327 * @param pVCpu Pointer to the VMCPU.
5328 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5329 * out-of-sync. Make sure to update the required fields
5330 * before using them.
5331 *
5332 * @remarks No-long-jump zone!!!
5333 */
5334static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5335{
5336 uint32_t uIntrState = 0;
5337 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5338 AssertRC(rc);
5339
5340 if (!uIntrState)
5341 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5342 else
5343 {
5344 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5345 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5346 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5347 AssertRC(rc);
5348 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5349 AssertRC(rc);
5350
5351 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5352 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5353 }
5354}
5355
5356
5357/**
5358 * Saves the guest's activity state.
5359 *
5360 * @returns VBox status code.
5361 * @param pVCpu Pointer to the VMCPU.
5362 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5363 * out-of-sync. Make sure to update the required fields
5364 * before using them.
5365 *
5366 * @remarks No-long-jump zone!!!
5367 */
5368static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5369{
5370 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5371 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5372 return VINF_SUCCESS;
5373}
5374
5375
5376/**
5377 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5378 * the current VMCS into the guest-CPU context.
5379 *
5380 * @returns VBox status code.
5381 * @param pVCpu Pointer to the VMCPU.
5382 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5383 * out-of-sync. Make sure to update the required fields
5384 * before using them.
5385 *
5386 * @remarks No-long-jump zone!!!
5387 */
5388static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5389{
5390 int rc = VINF_SUCCESS;
5391 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5392 {
5393 uint32_t u32Val = 0;
5394 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5395 pMixedCtx->SysEnter.cs = u32Val;
5396 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5397 }
5398
5399 uint64_t u64Val = 0;
5400 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5401 {
5402 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5403 pMixedCtx->SysEnter.eip = u64Val;
5404 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5405 }
5406 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5407 {
5408 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5409 pMixedCtx->SysEnter.esp = u64Val;
5410 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5411 }
5412 return rc;
5413}
5414
5415
5416/**
5417 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5418 * context.
5419 *
5420 * @returns VBox status code.
5421 * @param pVCpu Pointer to the VMCPU.
5422 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5423 * out-of-sync. Make sure to update the required fields
5424 * before using them.
5425 *
5426 * @remarks No-long-jump zone!!!
5427 */
5428static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5429{
5430 int rc = VINF_SUCCESS;
5431 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5432 {
5433 uint64_t u64Val = 0;
5434 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5435 pMixedCtx->fs.u64Base = u64Val;
5436 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5437 }
5438 return rc;
5439}
5440
5441
5442/**
5443 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5444 * context.
5445 *
5446 * @returns VBox status code.
5447 * @param pVCpu Pointer to the VMCPU.
5448 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5449 * out-of-sync. Make sure to update the required fields
5450 * before using them.
5451 *
5452 * @remarks No-long-jump zone!!!
5453 */
5454static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5455{
5456 int rc = VINF_SUCCESS;
5457 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5458 {
5459 uint64_t u64Val = 0;
5460 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5461 pMixedCtx->gs.u64Base = u64Val;
5462 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5463 }
5464 return rc;
5465}
5466
5467
5468/**
5469 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5470 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5471 * and TSC_AUX.
5472 *
5473 * @returns VBox status code.
5474 * @param pVCpu Pointer to the VMCPU.
5475 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5476 * out-of-sync. Make sure to update the required fields
5477 * before using them.
5478 *
5479 * @remarks No-long-jump zone!!!
5480 */
5481static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5482{
5483 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5484 return VINF_SUCCESS;
5485
5486#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5487 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5488 {
5489 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5490 pMsr += i;
5491 switch (pMsr->u32Msr)
5492 {
5493 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5494 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5495 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5496 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5497 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5498 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5499 default:
5500 {
5501 AssertFailed();
5502 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5503 }
5504 }
5505 }
5506#endif
5507
5508 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5509 return VINF_SUCCESS;
5510}
5511
5512
5513/**
5514 * Saves the guest control registers from the current VMCS into the guest-CPU
5515 * context.
5516 *
5517 * @returns VBox status code.
5518 * @param pVCpu Pointer to the VMCPU.
5519 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5520 * out-of-sync. Make sure to update the required fields
5521 * before using them.
5522 *
5523 * @remarks No-long-jump zone!!!
5524 */
5525static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5526{
5527 /* Guest CR0. Guest FPU. */
5528 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5529 AssertRCReturn(rc, rc);
5530
5531 /* Guest CR4. */
5532 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5533 AssertRCReturn(rc, rc);
5534
5535 /* Guest CR2 - updated always during the world-switch or in #PF. */
5536 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5537 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5538 {
5539 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5540 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5541
5542 PVM pVM = pVCpu->CTX_SUFF(pVM);
5543 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5544 || ( pVM->hm.s.fNestedPaging
5545 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5546 {
5547 uint64_t u64Val = 0;
5548 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5549 if (pMixedCtx->cr3 != u64Val)
5550 {
5551 CPUMSetGuestCR3(pVCpu, u64Val);
5552 if (VMMRZCallRing3IsEnabled(pVCpu))
5553 {
5554 PGMUpdateCR3(pVCpu, u64Val);
5555 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5556 }
5557 else
5558 {
5559 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5560 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5561 }
5562 }
5563
5564 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5565 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5566 {
5567 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5568 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5569 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5570 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5571
5572 if (VMMRZCallRing3IsEnabled(pVCpu))
5573 {
5574 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5575 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5576 }
5577 else
5578 {
5579 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5580 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5581 }
5582 }
5583 }
5584
5585 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5586 }
5587
5588 /*
5589 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5590 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5591 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5592 *
5593 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5594 */
5595 if (VMMRZCallRing3IsEnabled(pVCpu))
5596 {
5597 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5598 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5599
5600 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5601 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5602
5603 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5604 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5605 }
5606
5607 return rc;
5608}
5609
5610
5611/**
5612 * Reads a guest segment register from the current VMCS into the guest-CPU
5613 * context.
5614 *
5615 * @returns VBox status code.
5616 * @param pVCpu Pointer to the VMCPU.
5617 * @param idxSel Index of the selector in the VMCS.
5618 * @param idxLimit Index of the segment limit in the VMCS.
5619 * @param idxBase Index of the segment base in the VMCS.
5620 * @param idxAccess Index of the access rights of the segment in the VMCS.
5621 * @param pSelReg Pointer to the segment selector.
5622 *
5623 * @remarks No-long-jump zone!!!
5624 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5625 * macro as that takes care of whether to read from the VMCS cache or
5626 * not.
5627 */
5628DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5629 PCPUMSELREG pSelReg)
5630{
5631 uint32_t u32Val = 0;
5632 int rc = VMXReadVmcs32(idxSel, &u32Val);
5633 AssertRCReturn(rc, rc);
5634 pSelReg->Sel = (uint16_t)u32Val;
5635 pSelReg->ValidSel = (uint16_t)u32Val;
5636 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5637
5638 rc = VMXReadVmcs32(idxLimit, &u32Val);
5639 AssertRCReturn(rc, rc);
5640 pSelReg->u32Limit = u32Val;
5641
5642 uint64_t u64Val = 0;
5643 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5644 AssertRCReturn(rc, rc);
5645 pSelReg->u64Base = u64Val;
5646
5647 rc = VMXReadVmcs32(idxAccess, &u32Val);
5648 AssertRCReturn(rc, rc);
5649 pSelReg->Attr.u = u32Val;
5650
5651 /*
5652 * If VT-x marks the segment as unusable, most other bits remain undefined:
5653 * - For CS the L, D and G bits have meaning.
5654 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5655 * - For the remaining data segments no bits are defined.
5656 *
5657 * The present bit and the unusable bit has been observed to be set at the
5658 * same time (the selector was supposed to invalid as we started executing
5659 * a V8086 interrupt in ring-0).
5660 *
5661 * What should be important for the rest of the VBox code that the P bit is
5662 * cleared. Some of the other VBox code recognizes the unusable bit, but
5663 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5664 * safe side here, we'll strip off P and other bits we don't care about. If
5665 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5666 *
5667 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5668 */
5669 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5670 {
5671 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5672
5673 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5674 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5675 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5676
5677 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5678#ifdef DEBUG_bird
5679 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5680 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5681 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5682#endif
5683 }
5684 return VINF_SUCCESS;
5685}
5686
5687
5688#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5689# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5690 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5691 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5692#else
5693# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5694 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5695 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5696#endif
5697
5698
5699/**
5700 * Saves the guest segment registers from the current VMCS into the guest-CPU
5701 * context.
5702 *
5703 * @returns VBox status code.
5704 * @param pVCpu Pointer to the VMCPU.
5705 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5706 * out-of-sync. Make sure to update the required fields
5707 * before using them.
5708 *
5709 * @remarks No-long-jump zone!!!
5710 */
5711static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5712{
5713 /* Guest segment registers. */
5714 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5715 {
5716 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5717 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5718 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5719 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5720 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5721 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5722 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5723
5724 /* Restore segment attributes for real-on-v86 mode hack. */
5725 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5726 {
5727 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5728 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5729 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5730 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5731 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5732 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5733 }
5734 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5735 }
5736
5737 return VINF_SUCCESS;
5738}
5739
5740
5741/**
5742 * Saves the guest descriptor table registers and task register from the current
5743 * VMCS into the guest-CPU context.
5744 *
5745 * @returns VBox status code.
5746 * @param pVCpu Pointer to the VMCPU.
5747 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5748 * out-of-sync. Make sure to update the required fields
5749 * before using them.
5750 *
5751 * @remarks No-long-jump zone!!!
5752 */
5753static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5754{
5755 int rc = VINF_SUCCESS;
5756
5757 /* Guest LDTR. */
5758 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5759 {
5760 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5761 AssertRCReturn(rc, rc);
5762 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5763 }
5764
5765 /* Guest GDTR. */
5766 uint64_t u64Val = 0;
5767 uint32_t u32Val = 0;
5768 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5769 {
5770 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5771 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5772 pMixedCtx->gdtr.pGdt = u64Val;
5773 pMixedCtx->gdtr.cbGdt = u32Val;
5774 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
5775 }
5776
5777 /* Guest IDTR. */
5778 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5779 {
5780 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5781 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5782 pMixedCtx->idtr.pIdt = u64Val;
5783 pMixedCtx->idtr.cbIdt = u32Val;
5784 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
5785 }
5786
5787 /* Guest TR. */
5788 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5789 {
5790 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5791 AssertRCReturn(rc, rc);
5792
5793 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5794 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5795 {
5796 rc = VMXLOCAL_READ_SEG(TR, tr);
5797 AssertRCReturn(rc, rc);
5798 }
5799 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
5800 }
5801 return rc;
5802}
5803
5804#undef VMXLOCAL_READ_SEG
5805
5806
5807/**
5808 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5809 * context.
5810 *
5811 * @returns VBox status code.
5812 * @param pVCpu Pointer to the VMCPU.
5813 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5814 * out-of-sync. Make sure to update the required fields
5815 * before using them.
5816 *
5817 * @remarks No-long-jump zone!!!
5818 */
5819static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5820{
5821 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5822 {
5823 if (!pVCpu->hm.s.fUsingHyperDR7)
5824 {
5825 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5826 uint32_t u32Val;
5827 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5828 pMixedCtx->dr[7] = u32Val;
5829 }
5830
5831 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
5832 }
5833 return VINF_SUCCESS;
5834}
5835
5836
5837/**
5838 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
5839 *
5840 * @returns VBox status code.
5841 * @param pVCpu Pointer to the VMCPU.
5842 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5843 * out-of-sync. Make sure to update the required fields
5844 * before using them.
5845 *
5846 * @remarks No-long-jump zone!!!
5847 */
5848static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5849{
5850 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5851 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
5852 return VINF_SUCCESS;
5853}
5854
5855
5856/**
5857 * Saves the entire guest state from the currently active VMCS into the
5858 * guest-CPU context. This essentially VMREADs all guest-data.
5859 *
5860 * @returns VBox status code.
5861 * @param pVCpu Pointer to the VMCPU.
5862 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5863 * out-of-sync. Make sure to update the required fields
5864 * before using them.
5865 */
5866static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5867{
5868 Assert(pVCpu);
5869 Assert(pMixedCtx);
5870
5871 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5872 return VINF_SUCCESS;
5873
5874 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5875 again on the ring-3 callback path, there is no real need to. */
5876 if (VMMRZCallRing3IsEnabled(pVCpu))
5877 VMMR0LogFlushDisable(pVCpu);
5878 else
5879 Assert(VMMR0IsLogFlushDisabled(pVCpu));
5880 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5881
5882 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5883 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5884
5885 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5886 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5887
5888 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5889 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5890
5891 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5892 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5893
5894 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5895 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5896
5897 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5898 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5899
5900 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5901 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5902
5903 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5904 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5905
5906 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5907 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5908
5909 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5910 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5911
5912 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5913 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5914
5915 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5916 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5917
5918 if (VMMRZCallRing3IsEnabled(pVCpu))
5919 VMMR0LogFlushEnable(pVCpu);
5920
5921 return rc;
5922}
5923
5924
5925/**
5926 * Check per-VM and per-VCPU force flag actions that require us to go back to
5927 * ring-3 for one reason or another.
5928 *
5929 * @returns VBox status code (information status code included).
5930 * @retval VINF_SUCCESS if we don't have any actions that require going back to
5931 * ring-3.
5932 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5933 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5934 * interrupts)
5935 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5936 * all EMTs to be in ring-3.
5937 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5938 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5939 * to the EM loop.
5940 *
5941 * @param pVM Pointer to the VM.
5942 * @param pVCpu Pointer to the VMCPU.
5943 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5944 * out-of-sync. Make sure to update the required fields
5945 * before using them.
5946 */
5947static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5948{
5949 Assert(VMMRZCallRing3IsEnabled(pVCpu));
5950
5951 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5952 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5953 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5954 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5955 {
5956 /* We need the control registers now, make sure the guest-CPU context is updated. */
5957 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5958 AssertRCReturn(rc3, rc3);
5959
5960 /* Pending HM CR3 sync. */
5961 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5962 {
5963 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5964 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
5965 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5966 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5967 }
5968
5969 /* Pending HM PAE PDPEs. */
5970 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5971 {
5972 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5973 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5974 }
5975
5976 /* Pending PGM C3 sync. */
5977 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5978 {
5979 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
5980 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5981 if (rc2 != VINF_SUCCESS)
5982 {
5983 AssertRC(rc2);
5984 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5985 return rc2;
5986 }
5987 }
5988
5989 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5990 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5991 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5992 {
5993 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5994 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5995 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5996 return rc2;
5997 }
5998
5999 /* Pending VM request packets, such as hardware interrupts. */
6000 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6001 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6002 {
6003 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6004 return VINF_EM_PENDING_REQUEST;
6005 }
6006
6007 /* Pending PGM pool flushes. */
6008 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6009 {
6010 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6011 return VINF_PGM_POOL_FLUSH_PENDING;
6012 }
6013
6014 /* Pending DMA requests. */
6015 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6016 {
6017 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6018 return VINF_EM_RAW_TO_R3;
6019 }
6020 }
6021
6022 return VINF_SUCCESS;
6023}
6024
6025
6026/**
6027 * Converts any TRPM trap into a pending HM event. This is typically used when
6028 * entering from ring-3 (not longjmp returns).
6029 *
6030 * @param pVCpu Pointer to the VMCPU.
6031 */
6032static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6033{
6034 Assert(TRPMHasTrap(pVCpu));
6035 Assert(!pVCpu->hm.s.Event.fPending);
6036
6037 uint8_t uVector;
6038 TRPMEVENT enmTrpmEvent;
6039 RTGCUINT uErrCode;
6040 RTGCUINTPTR GCPtrFaultAddress;
6041 uint8_t cbInstr;
6042
6043 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6044 AssertRC(rc);
6045
6046 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
6047 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6048 if (enmTrpmEvent == TRPM_TRAP)
6049 {
6050 switch (uVector)
6051 {
6052 case X86_XCPT_BP:
6053 case X86_XCPT_OF:
6054 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6055 break;
6056
6057 case X86_XCPT_PF:
6058 case X86_XCPT_DF:
6059 case X86_XCPT_TS:
6060 case X86_XCPT_NP:
6061 case X86_XCPT_SS:
6062 case X86_XCPT_GP:
6063 case X86_XCPT_AC:
6064 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6065 /* no break! */
6066 default:
6067 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6068 break;
6069 }
6070 }
6071 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6072 {
6073 if (uVector == X86_XCPT_NMI)
6074 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6075 else
6076 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6077 }
6078 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6079 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6080 else
6081 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6082
6083 rc = TRPMResetTrap(pVCpu);
6084 AssertRC(rc);
6085 Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6086 u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6087
6088 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6089 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6090}
6091
6092
6093/**
6094 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6095 * VT-x to execute any instruction.
6096 *
6097 * @param pvCpu Pointer to the VMCPU.
6098 */
6099static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6100{
6101 Assert(pVCpu->hm.s.Event.fPending);
6102
6103 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6104 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
6105 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
6106 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6107
6108 /* If a trap was already pending, we did something wrong! */
6109 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6110
6111 TRPMEVENT enmTrapType;
6112 switch (uVectorType)
6113 {
6114 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6115 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6116 enmTrapType = TRPM_HARDWARE_INT;
6117 break;
6118
6119 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6120 enmTrapType = TRPM_SOFTWARE_INT;
6121 break;
6122
6123 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6124 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6125 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6126 enmTrapType = TRPM_TRAP;
6127 break;
6128
6129 default:
6130 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6131 enmTrapType = TRPM_32BIT_HACK;
6132 break;
6133 }
6134
6135 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6136
6137 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6138 AssertRC(rc);
6139
6140 if (fErrorCodeValid)
6141 TRPMSetErrorCode(pVCpu, uErrorCode);
6142
6143 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6144 && uVector == X86_XCPT_PF)
6145 {
6146 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6147 }
6148 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6149 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6150 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6151 {
6152 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6153 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6154 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6155 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6156 }
6157 pVCpu->hm.s.Event.fPending = false;
6158}
6159
6160
6161/**
6162 * Does the necessary state syncing before returning to ring-3 for any reason
6163 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6164 *
6165 * @returns VBox status code.
6166 * @param pVM Pointer to the VM.
6167 * @param pVCpu Pointer to the VMCPU.
6168 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6169 * be out-of-sync. Make sure to update the required
6170 * fields before using them.
6171 * @param fSaveGuestState Whether to save the guest state or not.
6172 *
6173 * @remarks No-long-jmp zone!!!
6174 */
6175static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6176{
6177 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6178 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6179
6180 RTCPUID idCpu = RTMpCpuId();
6181 Log4Func(("HostCpuId=%u\n", idCpu));
6182
6183 /* Save the guest state if necessary. */
6184 if ( fSaveGuestState
6185 && pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6186 {
6187 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6188 AssertRCReturn(rc, rc);
6189 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6190 }
6191
6192 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6193 if (CPUMIsGuestFPUStateActive(pVCpu))
6194 {
6195 /* We shouldn't reload CR0 without saving it first. */
6196 if (!fSaveGuestState)
6197 {
6198 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6199 AssertRCReturn(rc, rc);
6200 }
6201 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6202 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6203 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6204 }
6205
6206 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6207#ifdef VBOX_STRICT
6208 if (CPUMIsHyperDebugStateActive(pVCpu))
6209 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6210#endif
6211 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6212 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6213 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6214 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6215
6216#if HC_ARCH_BITS == 64
6217 /* Restore host-state bits that VT-x only restores partially. */
6218 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6219 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6220 {
6221 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6222 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6223 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6224 }
6225#endif
6226
6227 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6228 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6229 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6230 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6231 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6232 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6233 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6234 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6235
6236 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6237
6238 /** @todo This kinda defeats the purpose of having preemption hooks.
6239 * The problem is, deregistering the hooks should be moved to a place that
6240 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6241 * context.
6242 */
6243 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6244 {
6245 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6246 AssertRCReturn(rc, rc);
6247
6248 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6249 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6250 }
6251 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6252 NOREF(idCpu);
6253
6254 return VINF_SUCCESS;
6255}
6256
6257
6258/**
6259 * Leaves the VT-x session.
6260 *
6261 * @returns VBox status code.
6262 * @param pVM Pointer to the VM.
6263 * @param pVCpu Pointer to the VMCPU.
6264 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6265 * out-of-sync. Make sure to update the required fields
6266 * before using them.
6267 *
6268 * @remarks No-long-jmp zone!!!
6269 */
6270DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6271{
6272 HM_DISABLE_PREEMPT_IF_NEEDED();
6273 HMVMX_ASSERT_CPU_SAFE();
6274 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6275 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6276
6277 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6278 and done this from the VMXR0ThreadCtxCallback(). */
6279 if (!pVCpu->hm.s.fLeaveDone)
6280 {
6281 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6282 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6283 pVCpu->hm.s.fLeaveDone = true;
6284 }
6285
6286 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6287 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6288 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6289 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6290 VMMR0ThreadCtxHooksDeregister(pVCpu);
6291
6292 /* Leave HM context. This takes care of local init (term). */
6293 int rc = HMR0LeaveCpu(pVCpu);
6294
6295 HM_RESTORE_PREEMPT_IF_NEEDED();
6296
6297 return rc;
6298}
6299
6300
6301/**
6302 * Does the necessary state syncing before doing a longjmp to ring-3.
6303 *
6304 * @returns VBox status code.
6305 * @param pVM Pointer to the VM.
6306 * @param pVCpu Pointer to the VMCPU.
6307 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6308 * out-of-sync. Make sure to update the required fields
6309 * before using them.
6310 *
6311 * @remarks No-long-jmp zone!!!
6312 */
6313DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6314{
6315 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6316}
6317
6318
6319/**
6320 * Take necessary actions before going back to ring-3.
6321 *
6322 * An action requires us to go back to ring-3. This function does the necessary
6323 * steps before we can safely return to ring-3. This is not the same as longjmps
6324 * to ring-3, this is voluntary and prepares the guest so it may continue
6325 * executing outside HM (recompiler/IEM).
6326 *
6327 * @returns VBox status code.
6328 * @param pVM Pointer to the VM.
6329 * @param pVCpu Pointer to the VMCPU.
6330 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6331 * out-of-sync. Make sure to update the required fields
6332 * before using them.
6333 * @param rcExit The reason for exiting to ring-3. Can be
6334 * VINF_VMM_UNKNOWN_RING3_CALL.
6335 */
6336static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6337{
6338 Assert(pVM);
6339 Assert(pVCpu);
6340 Assert(pMixedCtx);
6341 HMVMX_ASSERT_PREEMPT_SAFE();
6342
6343 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6344 {
6345 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6346 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6347 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6348 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6349 }
6350
6351 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6352 VMMRZCallRing3Disable(pVCpu);
6353 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6354
6355 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6356 if (pVCpu->hm.s.Event.fPending)
6357 {
6358 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6359 Assert(!pVCpu->hm.s.Event.fPending);
6360 }
6361
6362 /* Save guest state and restore host state bits. */
6363 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6364 AssertRCReturn(rc, rc);
6365 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6366
6367 /* Sync recompiler state. */
6368 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6369 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6370 | CPUM_CHANGED_LDTR
6371 | CPUM_CHANGED_GDTR
6372 | CPUM_CHANGED_IDTR
6373 | CPUM_CHANGED_TR
6374 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6375 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6376 if ( pVM->hm.s.fNestedPaging
6377 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6378 {
6379 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6380 }
6381
6382 /*
6383 * Clear the X86_EFL_TF if necessary.
6384 */
6385 if (pVCpu->hm.s.fClearTrapFlag)
6386 {
6387 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6388 pMixedCtx->eflags.Bits.u1TF = 0;
6389 pVCpu->hm.s.fClearTrapFlag = false;
6390 }
6391 /** @todo there seems to be issues with the resume flag when the monitor trap
6392 * flag is pending without being used. Seen early in bios init when
6393 * accessing APIC page in prot mode. */
6394
6395 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6396 if (rcExit != VINF_EM_RAW_INTERRUPT)
6397 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6398
6399 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6400
6401 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6402 VMMRZCallRing3RemoveNotification(pVCpu);
6403 VMMRZCallRing3Enable(pVCpu);
6404
6405 return rc;
6406}
6407
6408
6409/**
6410 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6411 * longjump to ring-3 and possibly get preempted.
6412 *
6413 * @returns VBox status code.
6414 * @param pVCpu Pointer to the VMCPU.
6415 * @param enmOperation The operation causing the ring-3 longjump.
6416 * @param pvUser Opaque pointer to the guest-CPU context. The data
6417 * may be out-of-sync. Make sure to update the required
6418 * fields before using them.
6419 */
6420DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6421{
6422 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6423 {
6424 VMMRZCallRing3RemoveNotification(pVCpu);
6425 HM_DISABLE_PREEMPT_IF_NEEDED();
6426
6427 /* If anything here asserts or fails, good luck. */
6428 if (CPUMIsGuestFPUStateActive(pVCpu))
6429 CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6430
6431 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6432
6433#if HC_ARCH_BITS == 64
6434 /* Restore host-state bits that VT-x only restores partially. */
6435 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6436 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6437 {
6438 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6439 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6440 }
6441#endif
6442 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6443 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6444 {
6445 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6446 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6447 }
6448
6449 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6450 VMMR0ThreadCtxHooksDeregister(pVCpu);
6451
6452 HMR0LeaveCpu(pVCpu);
6453 HM_RESTORE_PREEMPT_IF_NEEDED();
6454 return VINF_SUCCESS;
6455 }
6456
6457 Assert(pVCpu);
6458 Assert(pvUser);
6459 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6460 HMVMX_ASSERT_PREEMPT_SAFE();
6461
6462 VMMRZCallRing3Disable(pVCpu);
6463 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6464
6465 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6466 enmOperation));
6467
6468 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6469 AssertRCReturn(rc, rc);
6470
6471 VMMRZCallRing3Enable(pVCpu);
6472 return VINF_SUCCESS;
6473}
6474
6475
6476/**
6477 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6478 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6479 *
6480 * @param pVCpu Pointer to the VMCPU.
6481 */
6482DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6483{
6484 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6485 {
6486 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6487 {
6488 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6489 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6490 AssertRC(rc);
6491 }
6492 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6493}
6494
6495
6496/**
6497 * Evaluates the event to be delivered to the guest and sets it as the pending
6498 * event.
6499 *
6500 * @param pVCpu Pointer to the VMCPU.
6501 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6502 * out-of-sync. Make sure to update the required fields
6503 * before using them.
6504 */
6505static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6506{
6507 Assert(!pVCpu->hm.s.Event.fPending);
6508
6509 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6510 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6511 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6512 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6513
6514 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6515 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6516 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6517 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6518 Assert(!TRPMHasTrap(pVCpu));
6519
6520 /** @todo SMI. SMIs take priority over NMIs. */
6521 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6522 {
6523 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6524 if ( !fBlockMovSS
6525 && !fBlockSti)
6526 {
6527 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6528 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6529 uint32_t u32IntrInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6530 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6531
6532 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6533 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6534 }
6535 else
6536 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6537 }
6538 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6539 && !pVCpu->hm.s.fSingleInstruction)
6540 {
6541 /*
6542 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6543 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6544 * evaluated here and not set as pending, solely based on the force-flags.
6545 */
6546 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6547 AssertRC(rc);
6548 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6549 if ( !fBlockInt
6550 && !fBlockSti
6551 && !fBlockMovSS)
6552 {
6553 uint8_t u8Interrupt;
6554 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6555 if (RT_SUCCESS(rc))
6556 {
6557 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6558 uint32_t u32IntrInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6559 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6560
6561 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6562 }
6563 else
6564 {
6565 /** @todo Does this actually happen? If not turn it into an assertion. */
6566 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6567 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6568 }
6569 }
6570 else
6571 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6572 }
6573}
6574
6575
6576/**
6577 * Injects any pending events into the guest if the guest is in a state to
6578 * receive them.
6579 *
6580 * @returns VBox status code (informational status codes included).
6581 * @param pVCpu Pointer to the VMCPU.
6582 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6583 * out-of-sync. Make sure to update the required fields
6584 * before using them.
6585 */
6586static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6587{
6588 HMVMX_ASSERT_PREEMPT_SAFE();
6589 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6590
6591 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6592 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6593 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6594 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6595
6596 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6597 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6598 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6599 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6600 Assert(!TRPMHasTrap(pVCpu));
6601
6602 int rc = VINF_SUCCESS;
6603 if (pVCpu->hm.s.Event.fPending)
6604 {
6605#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6606 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6607 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6608 {
6609 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6610 AssertRCReturn(rc, rc);
6611 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6612 Assert(!fBlockInt);
6613 Assert(!fBlockSti);
6614 Assert(!fBlockMovSS);
6615 }
6616 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6617 {
6618 Assert(!fBlockSti);
6619 Assert(!fBlockMovSS);
6620 }
6621#endif
6622 Log4(("Injecting pending event vcpu[%RU32] u64IntrInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo));
6623 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6624 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6625 AssertRCReturn(rc, rc);
6626
6627 /* Update the interruptibility-state as it could have been changed by
6628 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6629 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6630 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6631
6632#ifdef VBOX_WITH_STATISTICS
6633 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6634 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6635 else
6636 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6637#endif
6638 }
6639
6640 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6641 int rc2 = VINF_SUCCESS;
6642 if ( fBlockSti
6643 || fBlockMovSS)
6644 {
6645 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6646 {
6647 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6648 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6649 {
6650 /*
6651 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6652 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6653 * See Intel spec. 27.3.4 "Saving Non-Register State".
6654 */
6655 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6656 AssertRCReturn(rc, rc);
6657 }
6658 }
6659 else
6660 {
6661 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6662 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6663 uIntrState = 0;
6664 }
6665 }
6666
6667 /*
6668 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6669 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6670 */
6671 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6672 AssertRC(rc2);
6673
6674 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6675 return rc;
6676}
6677
6678
6679/**
6680 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6681 *
6682 * @param pVCpu Pointer to the VMCPU.
6683 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6684 * out-of-sync. Make sure to update the required fields
6685 * before using them.
6686 */
6687DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6688{
6689 uint32_t u32IntrInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6690 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6691}
6692
6693
6694/**
6695 * Injects a double-fault (#DF) exception into the VM.
6696 *
6697 * @returns VBox status code (informational status code included).
6698 * @param pVCpu Pointer to the VMCPU.
6699 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6700 * out-of-sync. Make sure to update the required fields
6701 * before using them.
6702 */
6703DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6704{
6705 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6706 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6707 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6708 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6709 puIntrState);
6710}
6711
6712
6713/**
6714 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6715 *
6716 * @param pVCpu Pointer to the VMCPU.
6717 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6718 * out-of-sync. Make sure to update the required fields
6719 * before using them.
6720 */
6721DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6722{
6723 uint32_t u32IntrInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6724 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6725 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6726}
6727
6728
6729/**
6730 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6731 *
6732 * @param pVCpu Pointer to the VMCPU.
6733 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6734 * out-of-sync. Make sure to update the required fields
6735 * before using them.
6736 * @param cbInstr The value of RIP that is to be pushed on the guest
6737 * stack.
6738 */
6739DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6740{
6741 uint32_t u32IntrInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6742 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6743 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6744}
6745
6746
6747/**
6748 * Injects a general-protection (#GP) fault into the VM.
6749 *
6750 * @returns VBox status code (informational status code included).
6751 * @param pVCpu Pointer to the VMCPU.
6752 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6753 * out-of-sync. Make sure to update the required fields
6754 * before using them.
6755 * @param u32ErrorCode The error code associated with the #GP.
6756 */
6757DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6758 uint32_t *puIntrState)
6759{
6760 uint32_t u32IntrInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6761 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6762 if (fErrorCodeValid)
6763 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6764 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6765 puIntrState);
6766}
6767
6768
6769/**
6770 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6771 *
6772 * @param pVCpu Pointer to the VMCPU.
6773 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6774 * out-of-sync. Make sure to update the required fields
6775 * before using them.
6776 * @param uVector The software interrupt vector number.
6777 * @param cbInstr The value of RIP that is to be pushed on the guest
6778 * stack.
6779 */
6780DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6781{
6782 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6783 if ( uVector == X86_XCPT_BP
6784 || uVector == X86_XCPT_OF)
6785 {
6786 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6787 }
6788 else
6789 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6790 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6791}
6792
6793
6794/**
6795 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6796 * stack.
6797 *
6798 * @returns VBox status code (information status code included).
6799 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6800 * @param pVM Pointer to the VM.
6801 * @param pMixedCtx Pointer to the guest-CPU context.
6802 * @param uValue The value to push to the guest stack.
6803 */
6804DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6805{
6806 /*
6807 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6808 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6809 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6810 */
6811 if (pMixedCtx->sp == 1)
6812 return VINF_EM_RESET;
6813 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6814 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6815 AssertRCReturn(rc, rc);
6816 return rc;
6817}
6818
6819
6820/**
6821 * Injects an event into the guest upon VM-entry by updating the relevant fields
6822 * in the VM-entry area in the VMCS.
6823 *
6824 * @returns VBox status code (informational error codes included).
6825 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6826 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6827 *
6828 * @param pVCpu Pointer to the VMCPU.
6829 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6830 * be out-of-sync. Make sure to update the required
6831 * fields before using them.
6832 * @param u64IntrInfo The VM-entry interruption-information field.
6833 * @param cbInstr The VM-entry instruction length in bytes (for
6834 * software interrupts, exceptions and privileged
6835 * software exceptions).
6836 * @param u32ErrCode The VM-entry exception error code.
6837 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6838 * @param puIntrState Pointer to the current guest interruptibility-state.
6839 * This interruptibility-state will be updated if
6840 * necessary. This cannot not be NULL.
6841 *
6842 * @remarks Requires CR0!
6843 * @remarks No-long-jump zone!!!
6844 */
6845static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6846 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6847{
6848 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6849 AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6850 Assert(puIntrState);
6851 uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6852
6853 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6854 const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6855
6856#ifdef VBOX_STRICT
6857 /* Validate the error-code-valid bit for hardware exceptions. */
6858 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6859 {
6860 switch (uVector)
6861 {
6862 case X86_XCPT_PF:
6863 case X86_XCPT_DF:
6864 case X86_XCPT_TS:
6865 case X86_XCPT_NP:
6866 case X86_XCPT_SS:
6867 case X86_XCPT_GP:
6868 case X86_XCPT_AC:
6869 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6870 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6871 /* fallthru */
6872 default:
6873 break;
6874 }
6875 }
6876#endif
6877
6878 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
6879 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6880 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6881
6882 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6883
6884 /* We require CR0 to check if the guest is in real-mode. */
6885 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6886 AssertRCReturn(rc, rc);
6887
6888 /*
6889 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6890 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6891 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6892 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6893 */
6894 if (CPUMIsGuestInRealModeEx(pMixedCtx))
6895 {
6896 PVM pVM = pVCpu->CTX_SUFF(pVM);
6897 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6898 {
6899 Assert(PDMVmmDevHeapIsEnabled(pVM));
6900 Assert(pVM->hm.s.vmx.pRealModeTSS);
6901
6902 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6903 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6904 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6905 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6906 AssertRCReturn(rc, rc);
6907 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6908
6909 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6910 const size_t cbIdtEntry = sizeof(X86IDTR16);
6911 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6912 {
6913 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6914 if (uVector == X86_XCPT_DF)
6915 return VINF_EM_RESET;
6916 else if (uVector == X86_XCPT_GP)
6917 {
6918 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6919 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6920 }
6921
6922 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6923 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6924 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
6925 }
6926
6927 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6928 uint16_t uGuestIp = pMixedCtx->ip;
6929 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6930 {
6931 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
6932 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
6933 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6934 }
6935 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6936 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6937
6938 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6939 X86IDTR16 IdtEntry;
6940 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6941 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6942 AssertRCReturn(rc, rc);
6943
6944 /* Construct the stack frame for the interrupt/exception handler. */
6945 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6946 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6947 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6948 AssertRCReturn(rc, rc);
6949
6950 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
6951 if (rc == VINF_SUCCESS)
6952 {
6953 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
6954 pMixedCtx->rip = IdtEntry.offSel;
6955 pMixedCtx->cs.Sel = IdtEntry.uSel;
6956 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6957 if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6958 && uVector == X86_XCPT_PF)
6959 {
6960 pMixedCtx->cr2 = GCPtrFaultAddress;
6961 }
6962
6963 /* If any other guest-state bits are changed here, make sure to update
6964 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6965 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
6966 | HM_CHANGED_GUEST_RIP
6967 | HM_CHANGED_GUEST_RFLAGS
6968 | HM_CHANGED_GUEST_RSP);
6969
6970 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6971 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6972 {
6973 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6974 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6975 Log4(("Clearing inhibition due to STI.\n"));
6976 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6977 }
6978 Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6979
6980 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
6981 it, if we are returning to ring-3 before executing guest code. */
6982 pVCpu->hm.s.Event.fPending = false;
6983 }
6984 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6985 return rc;
6986 }
6987 else
6988 {
6989 /*
6990 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6991 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6992 */
6993 u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6994 }
6995 }
6996
6997 /* Validate. */
6998 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6999 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
7000 Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7001
7002 /* Inject. */
7003 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
7004 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
7005 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7006 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7007
7008 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7009 && uVector == X86_XCPT_PF)
7010 {
7011 pMixedCtx->cr2 = GCPtrFaultAddress;
7012 }
7013
7014 Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7015 u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7016
7017 AssertRCReturn(rc, rc);
7018 return rc;
7019}
7020
7021
7022/**
7023 * Clears the interrupt-window exiting control in the VMCS and if necessary
7024 * clears the current event in the VMCS as well.
7025 *
7026 * @returns VBox status code.
7027 * @param pVCpu Pointer to the VMCPU.
7028 *
7029 * @remarks Use this function only to clear events that have not yet been
7030 * delivered to the guest but are injected in the VMCS!
7031 * @remarks No-long-jump zone!!!
7032 */
7033static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7034{
7035 int rc;
7036 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7037
7038 /* Clear interrupt-window exiting control. */
7039 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7040 {
7041 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7042 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7043 AssertRC(rc);
7044 }
7045
7046 if (!pVCpu->hm.s.Event.fPending)
7047 return;
7048
7049#ifdef VBOX_STRICT
7050 uint32_t u32EntryInfo;
7051 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7052 AssertRC(rc);
7053 Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
7054#endif
7055
7056 /* Clear the entry-interruption field (including the valid bit). */
7057 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7058 AssertRC(rc);
7059
7060 /* Clear the pending debug exception field. */
7061 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7062 AssertRC(rc);
7063}
7064
7065
7066/**
7067 * Enters the VT-x session.
7068 *
7069 * @returns VBox status code.
7070 * @param pVM Pointer to the VM.
7071 * @param pVCpu Pointer to the VMCPU.
7072 * @param pCpu Pointer to the CPU info struct.
7073 */
7074VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7075{
7076 AssertPtr(pVM);
7077 AssertPtr(pVCpu);
7078 Assert(pVM->hm.s.vmx.fSupported);
7079 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7080 NOREF(pCpu);
7081
7082 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7083 Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7084
7085#ifdef VBOX_STRICT
7086 /* Make sure we're in VMX root mode. */
7087 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7088 if (!(u32HostCR4 & X86_CR4_VMXE))
7089 {
7090 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7091 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7092 }
7093#endif
7094
7095 /*
7096 * Load the VCPU's VMCS as the current (and active) one.
7097 */
7098 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7099 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7100 if (RT_FAILURE(rc))
7101 return rc;
7102
7103 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7104 pVCpu->hm.s.fLeaveDone = false;
7105 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7106
7107 return VINF_SUCCESS;
7108}
7109
7110
7111/**
7112 * The thread-context callback (only on platforms which support it).
7113 *
7114 * @param enmEvent The thread-context event.
7115 * @param pVCpu Pointer to the VMCPU.
7116 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7117 * @thread EMT.
7118 */
7119VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7120{
7121 switch (enmEvent)
7122 {
7123 case RTTHREADCTXEVENT_PREEMPTING:
7124 {
7125 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7126 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7127 VMCPU_ASSERT_EMT(pVCpu);
7128
7129 PVM pVM = pVCpu->CTX_SUFF(pVM);
7130 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7131
7132 /* No longjmps (logger flushes, locks) in this fragile context. */
7133 VMMRZCallRing3Disable(pVCpu);
7134 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7135
7136 /*
7137 * Restore host-state (FPU, debug etc.)
7138 */
7139 if (!pVCpu->hm.s.fLeaveDone)
7140 {
7141 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7142 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7143 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7144 pVCpu->hm.s.fLeaveDone = true;
7145 }
7146
7147 /* Leave HM context, takes care of local init (term). */
7148 int rc = HMR0LeaveCpu(pVCpu);
7149 AssertRC(rc); NOREF(rc);
7150
7151 /* Restore longjmp state. */
7152 VMMRZCallRing3Enable(pVCpu);
7153 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7154 break;
7155 }
7156
7157 case RTTHREADCTXEVENT_RESUMED:
7158 {
7159 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7160 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7161 VMCPU_ASSERT_EMT(pVCpu);
7162
7163 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7164 VMMRZCallRing3Disable(pVCpu);
7165 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7166
7167 /* Initialize the bare minimum state required for HM. This takes care of
7168 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7169 int rc = HMR0EnterCpu(pVCpu);
7170 AssertRC(rc);
7171 Assert(VMCPU_HMCF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7172
7173 /* Load the active VMCS as the current one. */
7174 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7175 {
7176 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7177 AssertRC(rc); NOREF(rc);
7178 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7179 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7180 }
7181 pVCpu->hm.s.fLeaveDone = false;
7182
7183 /* Restore longjmp state. */
7184 VMMRZCallRing3Enable(pVCpu);
7185 break;
7186 }
7187
7188 default:
7189 break;
7190 }
7191}
7192
7193
7194/**
7195 * Saves the host state in the VMCS host-state.
7196 * Sets up the VM-exit MSR-load area.
7197 *
7198 * The CPU state will be loaded from these fields on every successful VM-exit.
7199 *
7200 * @returns VBox status code.
7201 * @param pVM Pointer to the VM.
7202 * @param pVCpu Pointer to the VMCPU.
7203 *
7204 * @remarks No-long-jump zone!!!
7205 */
7206static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7207{
7208 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7209
7210 if (!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7211 return VINF_SUCCESS;
7212
7213 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7214 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7215
7216 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7217 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7218
7219 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7220 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7221
7222 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7223 return rc;
7224}
7225
7226
7227/**
7228 * Saves the host state in the VMCS host-state.
7229 *
7230 * @returns VBox status code.
7231 * @param pVM Pointer to the VM.
7232 * @param pVCpu Pointer to the VMCPU.
7233 *
7234 * @remarks No-long-jump zone!!!
7235 */
7236VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7237{
7238 AssertPtr(pVM);
7239 AssertPtr(pVCpu);
7240
7241 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7242
7243 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7244 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7245 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7246 return hmR0VmxSaveHostState(pVM, pVCpu);
7247}
7248
7249
7250/**
7251 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7252 * loaded from these fields on every successful VM-entry.
7253 *
7254 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7255 * Sets up the VM-entry controls.
7256 * Sets up the appropriate VMX non-root function to execute guest code based on
7257 * the guest CPU mode.
7258 *
7259 * @returns VBox status code.
7260 * @param pVM Pointer to the VM.
7261 * @param pVCpu Pointer to the VMCPU.
7262 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7263 * out-of-sync. Make sure to update the required fields
7264 * before using them.
7265 *
7266 * @remarks No-long-jump zone!!!
7267 */
7268static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7269{
7270 AssertPtr(pVM);
7271 AssertPtr(pVCpu);
7272 AssertPtr(pMixedCtx);
7273 HMVMX_ASSERT_PREEMPT_SAFE();
7274
7275#ifdef LOG_ENABLED
7276 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7277 * probably not initialized yet? Anyway this will do for now.
7278 *
7279 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7280 * interface and disable ring-3 calls when thread-context hooks are not
7281 * available. */
7282 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7283 VMMR0LogFlushDisable(pVCpu);
7284#endif
7285
7286 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7287
7288 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7289
7290 /* Determine real-on-v86 mode. */
7291 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7292 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7293 && CPUMIsGuestInRealModeEx(pMixedCtx))
7294 {
7295 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7296 }
7297
7298 /*
7299 * Load the guest-state into the VMCS.
7300 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7301 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7302 */
7303 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7304 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7305
7306 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7307 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7308 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7309
7310 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7311 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7312 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7313
7314 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7315 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7316
7317 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7318 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7319
7320 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7321 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7322 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7323
7324 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7325 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7326
7327 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7328 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7329
7330 /*
7331 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7332 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7333 */
7334 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7335 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7336
7337 /* Clear any unused and reserved bits. */
7338 VMCPU_HMCF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
7339
7340#ifdef LOG_ENABLED
7341 /* Only reenable log-flushing if the caller has it enabled. */
7342 if (!fCallerDisabledLogFlush)
7343 VMMR0LogFlushEnable(pVCpu);
7344#endif
7345
7346 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7347 return rc;
7348}
7349
7350
7351/**
7352 * Loads the state shared between the host and guest into the VMCS.
7353 *
7354 * @param pVM Pointer to the VM.
7355 * @param pVCpu Pointer to the VMCPU.
7356 * @param pCtx Pointer to the guest-CPU context.
7357 *
7358 * @remarks No-long-jump zone!!!
7359 */
7360static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7361{
7362 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7363 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7364
7365 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
7366 {
7367 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7368 AssertRC(rc);
7369 }
7370
7371 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
7372 {
7373 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7374 AssertRC(rc);
7375
7376 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7377 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
7378 {
7379 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7380 AssertRC(rc);
7381 }
7382 }
7383
7384 AssertMsg(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
7385 ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7386}
7387
7388
7389/**
7390 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7391 *
7392 * @param pVM Pointer to the VM.
7393 * @param pVCpu Pointer to the VMCPU.
7394 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7395 * out-of-sync. Make sure to update the required fields
7396 * before using them.
7397 */
7398DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7399{
7400 HMVMX_ASSERT_PREEMPT_SAFE();
7401
7402 Log5(("LoadFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7403#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7404 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7405#endif
7406
7407 if (VMCPU_HMCF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
7408 {
7409 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7410 AssertRC(rc);
7411 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7412 }
7413 else if (VMCPU_HMCF_VALUE(pVCpu))
7414 {
7415 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7416 AssertRC(rc);
7417 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7418 }
7419
7420 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
7421 AssertMsg( !VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
7422 || VMCPU_HMCF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
7423 ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7424
7425#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7426 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7427 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7428 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7429#endif
7430}
7431
7432
7433/**
7434 * Does the preparations before executing guest code in VT-x.
7435 *
7436 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7437 * recompiler. We must be cautious what we do here regarding committing
7438 * guest-state information into the VMCS assuming we assuredly execute the
7439 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7440 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7441 * so that the recompiler can (and should) use them when it resumes guest
7442 * execution. Otherwise such operations must be done when we can no longer
7443 * exit to ring-3.
7444 *
7445 * @returns Strict VBox status code.
7446 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7447 * have been disabled.
7448 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7449 * double-fault into the guest.
7450 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7451 *
7452 * @param pVM Pointer to the VM.
7453 * @param pVCpu Pointer to the VMCPU.
7454 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7455 * out-of-sync. Make sure to update the required fields
7456 * before using them.
7457 * @param pVmxTransient Pointer to the VMX transient structure.
7458 *
7459 * @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7460 * interrupts will be disabled.
7461 */
7462static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7463{
7464 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7465
7466#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7467 PGMRZDynMapFlushAutoSet(pVCpu);
7468#endif
7469
7470 /* Check force flag actions that might require us to go back to ring-3. */
7471 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7472 if (rc != VINF_SUCCESS)
7473 return rc;
7474
7475#ifndef IEM_VERIFICATION_MODE_FULL
7476 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7477 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7478 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7479 {
7480 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7481 RTGCPHYS GCPhysApicBase;
7482 GCPhysApicBase = pMixedCtx->msrApicBase;
7483 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7484
7485 /* Unalias any existing mapping. */
7486 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7487 AssertRCReturn(rc, rc);
7488
7489 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7490 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7491 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7492 AssertRCReturn(rc, rc);
7493
7494 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7495 }
7496#endif /* !IEM_VERIFICATION_MODE_FULL */
7497
7498 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7499 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7500
7501 /*
7502 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7503 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7504 */
7505 if (TRPMHasTrap(pVCpu))
7506 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7507 else if (!pVCpu->hm.s.Event.fPending)
7508 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7509
7510 /*
7511 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7512 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7513 */
7514 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7515 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7516 {
7517 Assert(rc == VINF_EM_RESET);
7518 return rc;
7519 }
7520
7521 /*
7522 * No longjmps to ring-3 from this point on!!!
7523 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7524 * This also disables flushing of the R0-logger instance (if any).
7525 */
7526 VMMRZCallRing3Disable(pVCpu);
7527
7528 /*
7529 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7530 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7531 *
7532 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7533 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7534 *
7535 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7536 * executing guest code.
7537 */
7538 pVmxTransient->uEflags = ASMIntDisableFlags();
7539 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7540 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7541 {
7542 hmR0VmxClearEventVmcs(pVCpu);
7543 ASMSetFlags(pVmxTransient->uEflags);
7544 VMMRZCallRing3Enable(pVCpu);
7545 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7546 return VINF_EM_RAW_TO_R3;
7547 }
7548 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7549 {
7550 hmR0VmxClearEventVmcs(pVCpu);
7551 ASMSetFlags(pVmxTransient->uEflags);
7552 VMMRZCallRing3Enable(pVCpu);
7553 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7554 return VINF_EM_RAW_INTERRUPT;
7555 }
7556
7557 /* We've injected any pending events. This is really the point of no return (to ring-3). */
7558 pVCpu->hm.s.Event.fPending = false;
7559
7560 return VINF_SUCCESS;
7561}
7562
7563
7564/**
7565 * Prepares to run guest code in VT-x and we've committed to doing so. This
7566 * means there is no backing out to ring-3 or anywhere else at this
7567 * point.
7568 *
7569 * @param pVM Pointer to the VM.
7570 * @param pVCpu Pointer to the VMCPU.
7571 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7572 * out-of-sync. Make sure to update the required fields
7573 * before using them.
7574 * @param pVmxTransient Pointer to the VMX transient structure.
7575 *
7576 * @remarks Called with preemption disabled.
7577 * @remarks No-long-jump zone!!!
7578 */
7579static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7580{
7581 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7582 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7583 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7584
7585 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7586 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7587
7588 /*
7589 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7590 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7591 * Reload only the necessary state, the assertion will catch if other parts of the code
7592 * change.
7593 */
7594 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7595 {
7596 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7597 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7598 }
7599
7600#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7601 if (!CPUMIsGuestFPUStateActive(pVCpu))
7602 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7603 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7604#endif
7605
7606 /*
7607 * Load the host state bits as we may've been preempted (only happens when
7608 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7609 */
7610 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7611 {
7612 /* This ASSUMES that pfnStartVM has been set up already. */
7613 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7614 AssertRC(rc);
7615 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7616 }
7617 Assert(!VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
7618
7619 /*
7620 * Load the state shared between host and guest (FPU, debug).
7621 */
7622 if (VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
7623 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7624 AssertMsg(!VMCPU_HMCF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", VMCPU_HMCF_VALUE(pVCpu)));
7625
7626 /* Store status of the shared guest-host state at the time of VM-entry. */
7627#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7628 if (CPUMIsGuestInLongModeEx(pMixedCtx))
7629 {
7630 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7631 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7632 }
7633 else
7634#endif
7635 {
7636 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7637 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7638 }
7639 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7640
7641 /*
7642 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7643 */
7644 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7645 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7646
7647 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7648 RTCPUID idCurrentCpu = pCpu->idCpu;
7649 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7650 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
7651 {
7652 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7653 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7654 }
7655
7656 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7657 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7658 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7659 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7660
7661 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7662
7663 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7664 to start executing. */
7665
7666#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7667 /*
7668 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7669 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7670 */
7671 if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7672 && !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7673 {
7674 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7675 uint64_t u64HostTscAux = 0;
7676 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7677 AssertRC(rc2);
7678 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7679 }
7680#endif
7681}
7682
7683
7684/**
7685 * Performs some essential restoration of state after running guest code in
7686 * VT-x.
7687 *
7688 * @param pVM Pointer to the VM.
7689 * @param pVCpu Pointer to the VMCPU.
7690 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7691 * out-of-sync. Make sure to update the required fields
7692 * before using them.
7693 * @param pVmxTransient Pointer to the VMX transient structure.
7694 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7695 *
7696 * @remarks Called with interrupts disabled, and returns with interrups enabled!
7697 *
7698 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7699 * unconditionally when it is safe to do so.
7700 */
7701static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7702{
7703 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7704
7705 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7706 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7707 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7708 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7709 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7710
7711 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7712 {
7713#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7714 /* Restore host's TSC_AUX. */
7715 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7716 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7717#endif
7718 /** @todo Find a way to fix hardcoding a guestimate. */
7719 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7720 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7721 }
7722
7723 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7724 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7725 Assert(!(ASMGetFlags() & X86_EFL_IF));
7726 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7727
7728#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7729 if (CPUMIsGuestFPUStateActive(pVCpu))
7730 {
7731 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7732 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7733 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7734 }
7735#endif
7736
7737 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7738 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7739 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7740 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7741
7742 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7743 uint32_t uExitReason;
7744 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7745 rc |= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7746 AssertRC(rc);
7747 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7748 pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7749
7750 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7751 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7752 {
7753 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7754 pVmxTransient->fVMEntryFailed));
7755 return;
7756 }
7757
7758 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7759 {
7760 /* Update the guest interruptibility-state from the VMCS. */
7761 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7762#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7763 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7764 AssertRC(rc);
7765#endif
7766 /*
7767 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7768 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7769 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7770 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7771 */
7772 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7773 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7774 {
7775 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7776 AssertRC(rc);
7777 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
7778 }
7779 }
7780}
7781
7782
7783
7784/**
7785 * Runs the guest code using VT-x the normal way.
7786 *
7787 * @returns VBox status code.
7788 * @param pVM Pointer to the VM.
7789 * @param pVCpu Pointer to the VMCPU.
7790 * @param pCtx Pointer to the guest-CPU context.
7791 *
7792 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
7793 * @remarks Called with preemption disabled.
7794 */
7795static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7796{
7797 VMXTRANSIENT VmxTransient;
7798 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7799 int rc = VERR_INTERNAL_ERROR_5;
7800 uint32_t cLoops = 0;
7801
7802 for (;; cLoops++)
7803 {
7804 Assert(!HMR0SuspendPending());
7805 HMVMX_ASSERT_CPU_SAFE();
7806
7807 /* Preparatory work for running guest code, this may force us to return
7808 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7809 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7810 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7811 if (rc != VINF_SUCCESS)
7812 break;
7813
7814 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7815 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7816 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7817
7818 /* Restore any residual host-state and save any bits shared between host
7819 and guest into the guest-CPU state. Re-enables interrupts! */
7820 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7821
7822 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7823 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7824 {
7825 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7826 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7827 return rc;
7828 }
7829
7830 /* Handle the VM-exit. */
7831 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7832 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7833 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7834 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7835 HMVMX_START_EXIT_DISPATCH_PROF();
7836#ifdef HMVMX_USE_FUNCTION_TABLE
7837 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7838#else
7839 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7840#endif
7841 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7842 if (rc != VINF_SUCCESS)
7843 break;
7844 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7845 {
7846 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7847 rc = VINF_EM_RAW_INTERRUPT;
7848 break;
7849 }
7850 }
7851
7852 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7853 return rc;
7854}
7855
7856
7857/**
7858 * Single steps guest code using VT-x.
7859 *
7860 * @returns VBox status code.
7861 * @param pVM Pointer to the VM.
7862 * @param pVCpu Pointer to the VMCPU.
7863 * @param pCtx Pointer to the guest-CPU context.
7864 *
7865 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7866 * @remarks Called with preemption disabled.
7867 */
7868static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7869{
7870 VMXTRANSIENT VmxTransient;
7871 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7872 int rc = VERR_INTERNAL_ERROR_5;
7873 uint32_t cLoops = 0;
7874 uint16_t uCsStart = pCtx->cs.Sel;
7875 uint64_t uRipStart = pCtx->rip;
7876
7877 for (;; cLoops++)
7878 {
7879 Assert(!HMR0SuspendPending());
7880 HMVMX_ASSERT_CPU_SAFE();
7881
7882 /* Preparatory work for running guest code, this may force us to return
7883 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7884 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7885 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7886 if (rc != VINF_SUCCESS)
7887 break;
7888
7889 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7890 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7891 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7892
7893 /* Restore any residual host-state and save any bits shared between host
7894 and guest into the guest-CPU state. Re-enables interrupts! */
7895 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7896
7897 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7898 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7899 {
7900 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7901 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7902 return rc;
7903 }
7904
7905 /* Handle the VM-exit. */
7906 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7907 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7908 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7909 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7910 HMVMX_START_EXIT_DISPATCH_PROF();
7911#ifdef HMVMX_USE_FUNCTION_TABLE
7912 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7913#else
7914 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7915#endif
7916 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7917 if (rc != VINF_SUCCESS)
7918 break;
7919 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7920 {
7921 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7922 rc = VINF_EM_RAW_INTERRUPT;
7923 break;
7924 }
7925
7926 /*
7927 * Did the RIP change, if so, consider it a single step.
7928 * Otherwise, make sure one of the TFs gets set.
7929 */
7930 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7931 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7932 AssertRCReturn(rc2, rc2);
7933 if ( pCtx->rip != uRipStart
7934 || pCtx->cs.Sel != uCsStart)
7935 {
7936 rc = VINF_EM_DBG_STEPPED;
7937 break;
7938 }
7939 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7940 }
7941
7942 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7943 return rc;
7944}
7945
7946
7947/**
7948 * Runs the guest code using VT-x.
7949 *
7950 * @returns VBox status code.
7951 * @param pVM Pointer to the VM.
7952 * @param pVCpu Pointer to the VMCPU.
7953 * @param pCtx Pointer to the guest-CPU context.
7954 *
7955 * @remarks Called with preemption disabled.
7956 */
7957VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7958{
7959 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7960 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
7961 HMVMX_ASSERT_PREEMPT_SAFE();
7962
7963 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
7964
7965 int rc;
7966 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7967 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7968 else
7969 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7970
7971 if (rc == VERR_EM_INTERPRETER)
7972 rc = VINF_EM_RAW_EMULATE_INSTR;
7973 else if (rc == VINF_EM_RESET)
7974 rc = VINF_EM_TRIPLE_FAULT;
7975
7976 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7977 if (RT_FAILURE(rc2))
7978 {
7979 pVCpu->hm.s.u32HMError = rc;
7980 rc = rc2;
7981 }
7982 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
7983 return rc;
7984}
7985
7986
7987#ifndef HMVMX_USE_FUNCTION_TABLE
7988DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7989{
7990 int rc;
7991 switch (rcReason)
7992 {
7993 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7994 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7995 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7996 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7997 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7998 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7999 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8000 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
8001 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
8002 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
8003 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8004 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
8005 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
8006 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
8007 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
8008 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8009 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
8010 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
8011 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
8012 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
8013 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
8014 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
8015 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
8016 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
8017 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
8018 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8019 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
8020 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
8021 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
8022 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
8023 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
8024 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
8025 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
8026
8027 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8028 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8029 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8030 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8031 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8032 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8033 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8034 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8035 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8036
8037 case VMX_EXIT_VMCALL:
8038 case VMX_EXIT_VMCLEAR:
8039 case VMX_EXIT_VMLAUNCH:
8040 case VMX_EXIT_VMPTRLD:
8041 case VMX_EXIT_VMPTRST:
8042 case VMX_EXIT_VMREAD:
8043 case VMX_EXIT_VMRESUME:
8044 case VMX_EXIT_VMWRITE:
8045 case VMX_EXIT_VMXOFF:
8046 case VMX_EXIT_VMXON:
8047 case VMX_EXIT_INVEPT:
8048 case VMX_EXIT_INVVPID:
8049 case VMX_EXIT_VMFUNC:
8050 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8051 break;
8052 default:
8053 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8054 break;
8055 }
8056 return rc;
8057}
8058#endif
8059
8060#ifdef DEBUG
8061/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8062# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8063 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8064
8065# define HMVMX_ASSERT_PREEMPT_CPUID() \
8066 do \
8067 { \
8068 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8069 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8070 } while (0)
8071
8072# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8073 do { \
8074 AssertPtr(pVCpu); \
8075 AssertPtr(pMixedCtx); \
8076 AssertPtr(pVmxTransient); \
8077 Assert(pVmxTransient->fVMEntryFailed == false); \
8078 Assert(ASMIntAreEnabled()); \
8079 HMVMX_ASSERT_PREEMPT_SAFE(); \
8080 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8081 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8082 HMVMX_ASSERT_PREEMPT_SAFE(); \
8083 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8084 HMVMX_ASSERT_PREEMPT_CPUID(); \
8085 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8086 } while (0)
8087
8088# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8089 do { \
8090 Log4Func(("\n")); \
8091 } while(0)
8092#else /* Release builds */
8093# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8094# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8095#endif
8096
8097
8098/**
8099 * Advances the guest RIP after reading it from the VMCS.
8100 *
8101 * @returns VBox status code.
8102 * @param pVCpu Pointer to the VMCPU.
8103 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8104 * out-of-sync. Make sure to update the required fields
8105 * before using them.
8106 * @param pVmxTransient Pointer to the VMX transient structure.
8107 *
8108 * @remarks No-long-jump zone!!!
8109 */
8110DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8111{
8112 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8113 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8114 AssertRCReturn(rc, rc);
8115
8116 pMixedCtx->rip += pVmxTransient->cbInstr;
8117 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8118 return rc;
8119}
8120
8121
8122/**
8123 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8124 * and update error record fields accordingly.
8125 *
8126 * @return VMX_IGS_* return codes.
8127 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8128 * wrong with the guest state.
8129 *
8130 * @param pVM Pointer to the VM.
8131 * @param pVCpu Pointer to the VMCPU.
8132 * @param pCtx Pointer to the guest-CPU state.
8133 */
8134static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8135{
8136#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8137#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8138 uError = (err); \
8139 break; \
8140 } else do {} while (0)
8141/* Duplicate of IEM_IS_CANONICAL(). */
8142#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8143
8144 int rc;
8145 uint32_t uError = VMX_IGS_ERROR;
8146 uint32_t u32Val;
8147 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8148
8149 do
8150 {
8151 /*
8152 * CR0.
8153 */
8154 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8155 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8156 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8157 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8158 if (fUnrestrictedGuest)
8159 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8160
8161 uint32_t u32GuestCR0;
8162 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8163 AssertRCBreak(rc);
8164 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8165 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8166 if ( !fUnrestrictedGuest
8167 && (u32GuestCR0 & X86_CR0_PG)
8168 && !(u32GuestCR0 & X86_CR0_PE))
8169 {
8170 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8171 }
8172
8173 /*
8174 * CR4.
8175 */
8176 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8177 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8178
8179 uint32_t u32GuestCR4;
8180 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8181 AssertRCBreak(rc);
8182 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8183 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8184
8185 /*
8186 * IA32_DEBUGCTL MSR.
8187 */
8188 uint64_t u64Val;
8189 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8190 AssertRCBreak(rc);
8191 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8192 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8193 {
8194 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8195 }
8196 uint64_t u64DebugCtlMsr = u64Val;
8197
8198#ifdef VBOX_STRICT
8199 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8200 AssertRCBreak(rc);
8201 Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8202#endif
8203 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8204
8205 /*
8206 * RIP and RFLAGS.
8207 */
8208 uint32_t u32Eflags;
8209#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8210 if (HMVMX_IS_64BIT_HOST_MODE())
8211 {
8212 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8213 AssertRCBreak(rc);
8214 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8215 if ( !fLongModeGuest
8216 || !pCtx->cs.Attr.n.u1Long)
8217 {
8218 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8219 }
8220 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8221 * must be identical if the "IA32e mode guest" VM-entry control is 1
8222 * and CS.L is 1. No check applies if the CPU supports 64
8223 * linear-address bits. */
8224
8225 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8226 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8227 AssertRCBreak(rc);
8228 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8229 VMX_IGS_RFLAGS_RESERVED);
8230 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8231 u32Eflags = u64Val;
8232 }
8233 else
8234#endif
8235 {
8236 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8237 AssertRCBreak(rc);
8238 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8239 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8240 }
8241
8242 if ( fLongModeGuest
8243 || ( fUnrestrictedGuest
8244 && !(u32GuestCR0 & X86_CR0_PE)))
8245 {
8246 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8247 }
8248
8249 uint32_t u32EntryInfo;
8250 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8251 AssertRCBreak(rc);
8252 if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8253 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8254 {
8255 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8256 }
8257
8258 /*
8259 * 64-bit checks.
8260 */
8261#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8262 if (HMVMX_IS_64BIT_HOST_MODE())
8263 {
8264 if ( fLongModeGuest
8265 && !fUnrestrictedGuest)
8266 {
8267 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8268 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8269 }
8270
8271 if ( !fLongModeGuest
8272 && (u32GuestCR4 & X86_CR4_PCIDE))
8273 {
8274 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8275 }
8276
8277 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8278 * 51:32 beyond the processor's physical-address width are 0. */
8279
8280 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8281 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8282 {
8283 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8284 }
8285
8286 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8287 AssertRCBreak(rc);
8288 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8289
8290 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8291 AssertRCBreak(rc);
8292 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8293 }
8294#endif
8295
8296 /*
8297 * PERF_GLOBAL MSR.
8298 */
8299 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8300 {
8301 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8302 AssertRCBreak(rc);
8303 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8304 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8305 }
8306
8307 /*
8308 * PAT MSR.
8309 */
8310 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8311 {
8312 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8313 AssertRCBreak(rc);
8314 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8315 for (unsigned i = 0; i < 8; i++)
8316 {
8317 uint8_t u8Val = (u64Val & 0x7);
8318 if ( u8Val != 0 /* UC */
8319 || u8Val != 1 /* WC */
8320 || u8Val != 4 /* WT */
8321 || u8Val != 5 /* WP */
8322 || u8Val != 6 /* WB */
8323 || u8Val != 7 /* UC- */)
8324 {
8325 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8326 }
8327 u64Val >>= 3;
8328 }
8329 }
8330
8331 /*
8332 * EFER MSR.
8333 */
8334 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8335 {
8336 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8337 AssertRCBreak(rc);
8338 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8339 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8340 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8341 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8342 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8343 || (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8344 }
8345
8346 /*
8347 * Segment registers.
8348 */
8349 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8350 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8351 if (!(u32Eflags & X86_EFL_VM))
8352 {
8353 /* CS */
8354 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8355 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8356 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8357 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8358 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8359 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8360 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8361 /* CS cannot be loaded with NULL in protected mode. */
8362 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8363 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8364 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8365 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8366 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8367 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8368 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8369 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8370 else
8371 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8372
8373 /* SS */
8374 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8375 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8376 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8377 if ( !(pCtx->cr0 & X86_CR0_PE)
8378 || pCtx->cs.Attr.n.u4Type == 3)
8379 {
8380 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8381 }
8382 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8383 {
8384 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8385 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8386 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8387 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8388 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8389 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8390 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8391 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8392 }
8393
8394 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8395 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8396 {
8397 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8398 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8399 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8400 || pCtx->ds.Attr.n.u4Type > 11
8401 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8402 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8403 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8404 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8405 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8406 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8407 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8408 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8409 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8410 }
8411 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8412 {
8413 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8414 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8415 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8416 || pCtx->es.Attr.n.u4Type > 11
8417 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8418 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8419 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8420 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8421 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8422 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8423 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8424 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8425 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8426 }
8427 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8428 {
8429 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8430 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8431 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8432 || pCtx->fs.Attr.n.u4Type > 11
8433 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8434 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8435 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8436 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8437 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8438 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8439 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8440 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8441 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8442 }
8443 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8444 {
8445 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8446 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8447 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8448 || pCtx->gs.Attr.n.u4Type > 11
8449 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8450 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8451 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8452 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8453 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8454 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8455 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8456 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8457 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8458 }
8459 /* 64-bit capable CPUs. */
8460#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8461 if (HMVMX_IS_64BIT_HOST_MODE())
8462 {
8463 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8464 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8465 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8466 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8467 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8468 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8469 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8470 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8471 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8472 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8473 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8474 }
8475#endif
8476 }
8477 else
8478 {
8479 /* V86 mode checks. */
8480 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8481 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8482 {
8483 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8484 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8485 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8486 }
8487 else
8488 {
8489 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8490 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8491 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8492 }
8493
8494 /* CS */
8495 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8496 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8497 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8498 /* SS */
8499 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8500 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8501 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8502 /* DS */
8503 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8504 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8505 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8506 /* ES */
8507 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8508 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8509 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8510 /* FS */
8511 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8512 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8513 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8514 /* GS */
8515 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8516 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8517 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8518 /* 64-bit capable CPUs. */
8519#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8520 if (HMVMX_IS_64BIT_HOST_MODE())
8521 {
8522 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8523 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8524 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8525 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8526 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8527 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8528 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8529 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8530 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8531 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8532 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8533 }
8534#endif
8535 }
8536
8537 /*
8538 * TR.
8539 */
8540 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8541 /* 64-bit capable CPUs. */
8542#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8543 if (HMVMX_IS_64BIT_HOST_MODE())
8544 {
8545 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8546 }
8547#endif
8548 if (fLongModeGuest)
8549 {
8550 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8551 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8552 }
8553 else
8554 {
8555 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8556 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8557 VMX_IGS_TR_ATTR_TYPE_INVALID);
8558 }
8559 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8560 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8561 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8562 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8563 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8564 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8565 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8566 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8567
8568 /*
8569 * GDTR and IDTR.
8570 */
8571#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8572 if (HMVMX_IS_64BIT_HOST_MODE())
8573 {
8574 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8575 AssertRCBreak(rc);
8576 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8577
8578 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8579 AssertRCBreak(rc);
8580 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8581 }
8582#endif
8583
8584 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8585 AssertRCBreak(rc);
8586 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8587
8588 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8589 AssertRCBreak(rc);
8590 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8591
8592 /*
8593 * Guest Non-Register State.
8594 */
8595 /* Activity State. */
8596 uint32_t u32ActivityState;
8597 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8598 AssertRCBreak(rc);
8599 HMVMX_CHECK_BREAK( !u32ActivityState
8600 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8601 VMX_IGS_ACTIVITY_STATE_INVALID);
8602 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8603 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8604 uint32_t u32IntrState;
8605 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8606 AssertRCBreak(rc);
8607 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8608 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8609 {
8610 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8611 }
8612
8613 /** @todo Activity state and injecting interrupts. Left as a todo since we
8614 * currently don't use activity states but ACTIVE. */
8615
8616 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8617 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8618
8619 /* Guest interruptibility-state. */
8620 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8621 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8622 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8623 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8624 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8625 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8626 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8627 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8628 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8629 if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8630 {
8631 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8632 {
8633 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8634 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8635 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8636 }
8637 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8638 {
8639 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8640 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8641 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8642 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8643 }
8644 }
8645 /** @todo Assumes the processor is not in SMM. */
8646 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8647 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8648 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8649 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8650 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8651 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8652 && VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8653 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8654 {
8655 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8656 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8657 }
8658
8659 /* Pending debug exceptions. */
8660 if (HMVMX_IS_64BIT_HOST_MODE())
8661 {
8662 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8663 AssertRCBreak(rc);
8664 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8665 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8666 u32Val = u64Val; /* For pending debug exceptions checks below. */
8667 }
8668 else
8669 {
8670 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8671 AssertRCBreak(rc);
8672 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8673 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8674 }
8675
8676 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8677 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8678 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8679 {
8680 if ( (u32Eflags & X86_EFL_TF)
8681 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8682 {
8683 /* Bit 14 is PendingDebug.BS. */
8684 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8685 }
8686 if ( !(u32Eflags & X86_EFL_TF)
8687 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8688 {
8689 /* Bit 14 is PendingDebug.BS. */
8690 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8691 }
8692 }
8693
8694 /* VMCS link pointer. */
8695 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8696 AssertRCBreak(rc);
8697 if (u64Val != UINT64_C(0xffffffffffffffff))
8698 {
8699 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8700 /** @todo Bits beyond the processor's physical-address width MBZ. */
8701 /** @todo 32-bit located in memory referenced by value of this field (as a
8702 * physical address) must contain the processor's VMCS revision ID. */
8703 /** @todo SMM checks. */
8704 }
8705
8706 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8707
8708 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8709 if (uError == VMX_IGS_ERROR)
8710 uError = VMX_IGS_REASON_NOT_FOUND;
8711 } while (0);
8712
8713 pVCpu->hm.s.u32HMError = uError;
8714 return uError;
8715
8716#undef HMVMX_ERROR_BREAK
8717#undef HMVMX_CHECK_BREAK
8718#undef HMVMX_IS_CANONICAL
8719}
8720
8721/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8722/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8723/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8724
8725/** @name VM-exit handlers.
8726 * @{
8727 */
8728
8729/**
8730 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8731 */
8732HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8733{
8734 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8735 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8736 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8737#if HC_ARCH_BITS == 64
8738 Assert(ASMIntAreEnabled());
8739 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8740 return VINF_SUCCESS;
8741#endif
8742 return VINF_EM_RAW_INTERRUPT;
8743}
8744
8745
8746/**
8747 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8748 */
8749HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8750{
8751 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8752 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8753
8754 int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8755 AssertRCReturn(rc, rc);
8756
8757 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8758 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8759 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8760 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8761
8762 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8763 {
8764 /*
8765 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8766 * anything we inject is not going to cause a VM-exit directly for the event being injected.
8767 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8768 *
8769 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8770 */
8771 VMXDispatchHostNmi();
8772 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
8773 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8774 return VINF_SUCCESS;
8775 }
8776
8777 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8778 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8779 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8780 {
8781 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8782 return VINF_SUCCESS;
8783 }
8784 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8785 {
8786 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8787 return rc;
8788 }
8789
8790 uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8791 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8792 switch (uIntrType)
8793 {
8794 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8795 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8796 /* no break */
8797 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8798 {
8799 switch (uVector)
8800 {
8801 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8802 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8803 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8804 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8805 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8806 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8807#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8808 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8809 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8810 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8811 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8812 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8813 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8814 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8815 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8816 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8817 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8818#endif
8819 default:
8820 {
8821 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8822 AssertRCReturn(rc, rc);
8823
8824 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8825 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8826 {
8827 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8828 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8829 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8830
8831 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8832 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8833 AssertRCReturn(rc, rc);
8834 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8835 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8836 0 /* GCPtrFaultAddress */);
8837 AssertRCReturn(rc, rc);
8838 }
8839 else
8840 {
8841 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8842 pVCpu->hm.s.u32HMError = uVector;
8843 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8844 }
8845 break;
8846 }
8847 }
8848 break;
8849 }
8850
8851 default:
8852 {
8853 pVCpu->hm.s.u32HMError = uExitIntrInfo;
8854 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8855 AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8856 break;
8857 }
8858 }
8859 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8860 return rc;
8861}
8862
8863
8864/**
8865 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8866 */
8867HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8868{
8869 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8870
8871 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8872 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8873 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8874 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8875 AssertRCReturn(rc, rc);
8876
8877 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8878 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8879 return VINF_SUCCESS;
8880}
8881
8882
8883/**
8884 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8885 */
8886HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8887{
8888 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8889 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8890 pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8891 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8892}
8893
8894
8895/**
8896 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8897 */
8898HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8899{
8900 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8901 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8902 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8903}
8904
8905
8906/**
8907 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8908 */
8909HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8910{
8911 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8912 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8913 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8914}
8915
8916
8917/**
8918 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8919 */
8920HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8921{
8922 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8923 PVM pVM = pVCpu->CTX_SUFF(pVM);
8924 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8925 if (RT_LIKELY(rc == VINF_SUCCESS))
8926 {
8927 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8928 Assert(pVmxTransient->cbInstr == 2);
8929 }
8930 else
8931 {
8932 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8933 rc = VERR_EM_INTERPRETER;
8934 }
8935 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8936 return rc;
8937}
8938
8939
8940/**
8941 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8942 */
8943HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8944{
8945 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8946 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8947 AssertRCReturn(rc, rc);
8948
8949 if (pMixedCtx->cr4 & X86_CR4_SMXE)
8950 return VINF_EM_RAW_EMULATE_INSTR;
8951
8952 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8953 pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8954 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8955}
8956
8957
8958/**
8959 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8960 */
8961HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8962{
8963 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8964 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8965 AssertRCReturn(rc, rc);
8966
8967 PVM pVM = pVCpu->CTX_SUFF(pVM);
8968 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8969 if (RT_LIKELY(rc == VINF_SUCCESS))
8970 {
8971 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8972 Assert(pVmxTransient->cbInstr == 2);
8973 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8974 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8975 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8976 }
8977 else
8978 {
8979 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8980 rc = VERR_EM_INTERPRETER;
8981 }
8982 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8983 return rc;
8984}
8985
8986
8987/**
8988 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8989 */
8990HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8991{
8992 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8993 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8994 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8995 AssertRCReturn(rc, rc);
8996
8997 PVM pVM = pVCpu->CTX_SUFF(pVM);
8998 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8999 if (RT_LIKELY(rc == VINF_SUCCESS))
9000 {
9001 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9002 Assert(pVmxTransient->cbInstr == 3);
9003 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9004 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9005 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9006 }
9007 else
9008 {
9009 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9010 rc = VERR_EM_INTERPRETER;
9011 }
9012 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9013 return rc;
9014}
9015
9016
9017/**
9018 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9019 */
9020HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9021{
9022 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9023 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9024 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9025 AssertRCReturn(rc, rc);
9026
9027 PVM pVM = pVCpu->CTX_SUFF(pVM);
9028 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9029 if (RT_LIKELY(rc == VINF_SUCCESS))
9030 {
9031 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9032 Assert(pVmxTransient->cbInstr == 2);
9033 }
9034 else
9035 {
9036 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9037 rc = VERR_EM_INTERPRETER;
9038 }
9039 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9040 return rc;
9041}
9042
9043
9044/**
9045 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9046 */
9047HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9048{
9049 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9050 PVM pVM = pVCpu->CTX_SUFF(pVM);
9051 Assert(!pVM->hm.s.fNestedPaging);
9052
9053 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9054 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9055 AssertRCReturn(rc, rc);
9056
9057 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9058 rc = VBOXSTRICTRC_VAL(rc2);
9059 if (RT_LIKELY(rc == VINF_SUCCESS))
9060 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9061 else
9062 {
9063 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9064 pVmxTransient->uExitQualification, rc));
9065 }
9066 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9067 return rc;
9068}
9069
9070
9071/**
9072 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9073 */
9074HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9075{
9076 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9077 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9078 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9079 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9080 AssertRCReturn(rc, rc);
9081
9082 PVM pVM = pVCpu->CTX_SUFF(pVM);
9083 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9084 if (RT_LIKELY(rc == VINF_SUCCESS))
9085 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9086 else
9087 {
9088 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9089 rc = VERR_EM_INTERPRETER;
9090 }
9091 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9092 return rc;
9093}
9094
9095
9096/**
9097 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9098 */
9099HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9100{
9101 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9102 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9103 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9104 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9105 AssertRCReturn(rc, rc);
9106
9107 PVM pVM = pVCpu->CTX_SUFF(pVM);
9108 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9109 rc = VBOXSTRICTRC_VAL(rc2);
9110 if (RT_LIKELY( rc == VINF_SUCCESS
9111 || rc == VINF_EM_HALT))
9112 {
9113 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9114 AssertRCReturn(rc3, rc3);
9115
9116 if ( rc == VINF_EM_HALT
9117 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9118 {
9119 rc = VINF_SUCCESS;
9120 }
9121 }
9122 else
9123 {
9124 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9125 rc = VERR_EM_INTERPRETER;
9126 }
9127 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
9128 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9129 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9130 return rc;
9131}
9132
9133
9134/**
9135 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9136 */
9137HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9138{
9139 /*
9140 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9141 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9142 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
9143 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9144 */
9145 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9146 pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
9147 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9148}
9149
9150
9151/**
9152 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9153 */
9154HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9155{
9156 /*
9157 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9158 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9159 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9160 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9161 */
9162 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9163 pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
9164 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9165}
9166
9167
9168/**
9169 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9170 */
9171HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9172{
9173 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9174 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9175 pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
9176 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9177}
9178
9179
9180/**
9181 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9182 */
9183HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9184{
9185 /*
9186 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9187 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9188 * See Intel spec. 25.3 "Other Causes of VM-exits".
9189 */
9190 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9191 pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
9192 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9193}
9194
9195
9196/**
9197 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9198 * VM-exit.
9199 */
9200HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9201{
9202 /*
9203 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9204 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9205 *
9206 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9207 * See Intel spec. "23.8 Restrictions on VMX operation".
9208 */
9209 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9210 return VINF_SUCCESS;
9211}
9212
9213
9214/**
9215 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9216 * VM-exit.
9217 */
9218HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9219{
9220 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9221 return VINF_EM_RESET;
9222}
9223
9224
9225/**
9226 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9227 */
9228HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9229{
9230 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9231 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9232 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9233 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9234 AssertRCReturn(rc, rc);
9235
9236 pMixedCtx->rip++;
9237 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9238 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9239 rc = VINF_SUCCESS;
9240 else
9241 rc = VINF_EM_HALT;
9242
9243 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9244 return rc;
9245}
9246
9247
9248/**
9249 * VM-exit handler for instructions that result in a #UD exception delivered to
9250 * the guest.
9251 */
9252HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9253{
9254 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9255 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9256 return VINF_SUCCESS;
9257}
9258
9259
9260/**
9261 * VM-exit handler for expiry of the VMX preemption timer.
9262 */
9263HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9264{
9265 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9266
9267 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9268 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9269
9270 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9271 PVM pVM = pVCpu->CTX_SUFF(pVM);
9272 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9273 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9274 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9275}
9276
9277
9278/**
9279 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9280 */
9281HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9282{
9283 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9284
9285 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9286 /** @todo check if XSETBV is supported by the recompiler. */
9287 return VERR_EM_INTERPRETER;
9288}
9289
9290
9291/**
9292 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9293 */
9294HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9295{
9296 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9297
9298 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9299 /** @todo implement EMInterpretInvpcid() */
9300 return VERR_EM_INTERPRETER;
9301}
9302
9303
9304/**
9305 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9306 * Error VM-exit.
9307 */
9308HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9309{
9310 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9311 AssertRCReturn(rc, rc);
9312
9313 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9314 NOREF(uInvalidReason);
9315
9316#ifdef VBOX_STRICT
9317 uint32_t uIntrState;
9318 HMVMXHCUINTREG uHCReg;
9319 uint64_t u64Val;
9320 uint32_t u32Val;
9321
9322 rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
9323 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9324 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9325 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9326 AssertRCReturn(rc, rc);
9327
9328 Log4(("uInvalidReason %u\n", uInvalidReason));
9329 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
9330 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9331 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9332 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9333
9334 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9335 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9336 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9337 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9338 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9339 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9340 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9341 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9342 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9343 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9344 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9345 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9346#endif
9347
9348 PVM pVM = pVCpu->CTX_SUFF(pVM);
9349 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9350
9351 return VERR_VMX_INVALID_GUEST_STATE;
9352}
9353
9354
9355/**
9356 * VM-exit handler for VM-entry failure due to an MSR-load
9357 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9358 */
9359HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9360{
9361 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9362 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9363}
9364
9365
9366/**
9367 * VM-exit handler for VM-entry failure due to a machine-check event
9368 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9369 */
9370HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9371{
9372 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9373 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9374}
9375
9376
9377/**
9378 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9379 * theory.
9380 */
9381HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9382{
9383 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9384 return VERR_VMX_UNDEFINED_EXIT_CODE;
9385}
9386
9387
9388/**
9389 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9390 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9391 * Conditional VM-exit.
9392 */
9393HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9394{
9395 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9396
9397 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9398 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9399 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9400 return VERR_EM_INTERPRETER;
9401 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9402 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9403}
9404
9405
9406/**
9407 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9408 */
9409HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9410{
9411 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9412
9413 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9414 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9415 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9416 return VERR_EM_INTERPRETER;
9417 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9418 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9419}
9420
9421
9422/**
9423 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9424 */
9425HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9426{
9427 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9428
9429 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9430 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9431 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9432 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9433 AssertRCReturn(rc, rc);
9434 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9435
9436 PVM pVM = pVCpu->CTX_SUFF(pVM);
9437 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9438 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9439 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9440 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9441
9442 if (RT_LIKELY(rc == VINF_SUCCESS))
9443 {
9444 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9445 Assert(pVmxTransient->cbInstr == 2);
9446 }
9447 return rc;
9448}
9449
9450
9451/**
9452 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9453 */
9454HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9455{
9456 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9457 PVM pVM = pVCpu->CTX_SUFF(pVM);
9458 int rc = VINF_SUCCESS;
9459
9460 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9461 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9462 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9463 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9464 AssertRCReturn(rc, rc);
9465 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9466
9467 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9468 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9469 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9470
9471 if (RT_LIKELY(rc == VINF_SUCCESS))
9472 {
9473 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9474
9475 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9476 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9477 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9478 {
9479 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9480 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9481 EMInterpretWrmsr() changes it. */
9482 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9483 }
9484 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9485 {
9486 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9487 AssertRCReturn(rc, rc);
9488 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
9489 }
9490 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9491 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9492
9493 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9494 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9495 {
9496 switch (pMixedCtx->ecx)
9497 {
9498 case MSR_IA32_SYSENTER_CS: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
9499 case MSR_IA32_SYSENTER_EIP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
9500 case MSR_IA32_SYSENTER_ESP: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
9501 case MSR_K8_FS_BASE: /* no break */
9502 case MSR_K8_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
9503 case MSR_K8_KERNEL_GS_BASE: VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS); break;
9504 }
9505 }
9506#ifdef VBOX_STRICT
9507 else
9508 {
9509 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9510 switch (pMixedCtx->ecx)
9511 {
9512 case MSR_IA32_SYSENTER_CS:
9513 case MSR_IA32_SYSENTER_EIP:
9514 case MSR_IA32_SYSENTER_ESP:
9515 case MSR_K8_FS_BASE:
9516 case MSR_K8_GS_BASE:
9517 {
9518 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9519 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9520 }
9521
9522 case MSR_K8_LSTAR:
9523 case MSR_K6_STAR:
9524 case MSR_K8_SF_MASK:
9525 case MSR_K8_TSC_AUX:
9526 case MSR_K8_KERNEL_GS_BASE:
9527 {
9528 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9529 pMixedCtx->ecx));
9530 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9531 }
9532 }
9533 }
9534#endif /* VBOX_STRICT */
9535 }
9536 return rc;
9537}
9538
9539
9540/**
9541 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9542 */
9543HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9544{
9545 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9546
9547 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9548 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9549 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9550 return VERR_EM_INTERPRETER;
9551 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9552 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9553}
9554
9555
9556/**
9557 * VM-exit handler for when the TPR value is lowered below the specified
9558 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9559 */
9560HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9561{
9562 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9563 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9564
9565 /*
9566 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9567 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9568 * resume guest execution.
9569 */
9570 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9571 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9572 return VINF_SUCCESS;
9573}
9574
9575
9576/**
9577 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9578 * VM-exit.
9579 *
9580 * @retval VINF_SUCCESS when guest execution can continue.
9581 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9582 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9583 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9584 * recompiler.
9585 */
9586HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9587{
9588 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9589 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9590 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9591 AssertRCReturn(rc, rc);
9592
9593 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9594 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9595 PVM pVM = pVCpu->CTX_SUFF(pVM);
9596 switch (uAccessType)
9597 {
9598 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9599 {
9600#if 0
9601 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9602 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9603#else
9604 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9605 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9606 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9607#endif
9608 AssertRCReturn(rc, rc);
9609
9610 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9611 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9612 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9613 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9614
9615 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9616 {
9617 case 0: /* CR0 */
9618 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9619 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9620 break;
9621 case 2: /* C2 **/
9622 /* Nothing to do here, CR2 it's not part of the VMCS. */
9623 break;
9624 case 3: /* CR3 */
9625 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9626 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
9627 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9628 break;
9629 case 4: /* CR4 */
9630 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9631 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9632 break;
9633 case 8: /* CR8 */
9634 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9635 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9636 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9637 break;
9638 default:
9639 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9640 break;
9641 }
9642
9643 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9644 break;
9645 }
9646
9647 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9648 {
9649 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9650 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9651 AssertRCReturn(rc, rc);
9652 Assert( !pVM->hm.s.fNestedPaging
9653 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9654 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9655
9656 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9657 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9658 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9659
9660 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9661 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9662 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9663 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9664 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9665 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9666 break;
9667 }
9668
9669 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9670 {
9671 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9672 AssertRCReturn(rc, rc);
9673 rc = EMInterpretCLTS(pVM, pVCpu);
9674 AssertRCReturn(rc, rc);
9675 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9676 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9677 Log4(("CRX CLTS write rc=%d\n", rc));
9678 break;
9679 }
9680
9681 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9682 {
9683 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9684 AssertRCReturn(rc, rc);
9685 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9686 if (RT_LIKELY(rc == VINF_SUCCESS))
9687 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9688 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9689 Log4(("CRX LMSW write rc=%d\n", rc));
9690 break;
9691 }
9692
9693 default:
9694 {
9695 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9696 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9697 }
9698 }
9699
9700 /* Validate possible error codes. */
9701 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9702 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9703 if (RT_SUCCESS(rc))
9704 {
9705 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9706 AssertRCReturn(rc2, rc2);
9707 }
9708
9709 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9710 return rc;
9711}
9712
9713
9714/**
9715 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9716 * VM-exit.
9717 */
9718HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9719{
9720 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9721 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9722
9723 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9724 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9725 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9726 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9727 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9728 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9729 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9730 AssertRCReturn(rc2, rc2);
9731
9732 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9733 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9734 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9735 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9736 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9737 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9738 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9739
9740 /* I/O operation lookup arrays. */
9741 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9742 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9743
9744 VBOXSTRICTRC rcStrict;
9745 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9746 const uint32_t cbInstr = pVmxTransient->cbInstr;
9747 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9748 PVM pVM = pVCpu->CTX_SUFF(pVM);
9749 if (fIOString)
9750 {
9751 /*
9752 * INS/OUTS - I/O String instruction.
9753 *
9754 * Use instruction-information if available, otherwise fall back on
9755 * interpreting the instruction.
9756 */
9757 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9758#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9759 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9760 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9761 {
9762 rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9763 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9764 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9765 AssertRCReturn(rc2, rc2);
9766 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9767 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9768 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9769 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9770 if (fIOWrite)
9771 {
9772 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9773 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9774 //if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9775 // hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9776 // pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9777 }
9778 else
9779 {
9780 AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9781 ("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9782 VERR_HMVMX_IPE_4);
9783 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9784 //if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9785 // hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9786 }
9787 }
9788 else
9789 {
9790 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9791 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9792 AssertRCReturn(rc2, rc2);
9793 rcStrict = IEMExecOne(pVCpu);
9794 }
9795 /** @todo IEM needs to be setting these flags somehow. */
9796 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9797 fUpdateRipAlready = true;
9798#else
9799 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9800 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9801 if (RT_SUCCESS(rcStrict))
9802 {
9803 if (fIOWrite)
9804 {
9805 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9806 (DISCPUMODE)pDis->uAddrMode, cbValue);
9807 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9808 }
9809 else
9810 {
9811 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9812 (DISCPUMODE)pDis->uAddrMode, cbValue);
9813 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9814 }
9815 }
9816 else
9817 {
9818 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9819 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9820 }
9821#endif
9822 }
9823 else
9824 {
9825 /*
9826 * IN/OUT - I/O instruction.
9827 */
9828 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9829 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9830 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9831 if (fIOWrite)
9832 {
9833 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9834 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9835 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9837 }
9838 else
9839 {
9840 uint32_t u32Result = 0;
9841 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9842 if (IOM_SUCCESS(rcStrict))
9843 {
9844 /* Save result of I/O IN instr. in AL/AX/EAX. */
9845 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
9846 }
9847 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9848 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9849 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9850 }
9851 }
9852
9853 if (IOM_SUCCESS(rcStrict))
9854 {
9855 if (!fUpdateRipAlready)
9856 {
9857 pMixedCtx->rip += cbInstr;
9858 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9859 }
9860
9861 /*
9862 * If any I/O breakpoints are armed, we need to check if one triggered
9863 * and take appropriate action.
9864 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9865 */
9866 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9867 AssertRCReturn(rc2, rc2);
9868
9869 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9870 * execution engines about whether hyper BPs and such are pending. */
9871 uint32_t const uDr7 = pMixedCtx->dr[7];
9872 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9873 && X86_DR7_ANY_RW_IO(uDr7)
9874 && (pMixedCtx->cr4 & X86_CR4_DE))
9875 || DBGFBpIsHwIoArmed(pVM)))
9876 {
9877 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9878
9879 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
9880 VMMRZCallRing3Disable(pVCpu);
9881 HM_DISABLE_PREEMPT_IF_NEEDED();
9882
9883 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
9884
9885 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9886 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9887 {
9888 /* Raise #DB. */
9889 if (fIsGuestDbgActive)
9890 ASMSetDR6(pMixedCtx->dr[6]);
9891 if (pMixedCtx->dr[7] != uDr7)
9892 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
9893
9894 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9895 }
9896 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9897 else if ( rcStrict2 != VINF_SUCCESS
9898 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
9899 rcStrict = rcStrict2;
9900
9901 HM_RESTORE_PREEMPT_IF_NEEDED();
9902 VMMRZCallRing3Enable(pVCpu);
9903 }
9904 }
9905
9906#ifdef DEBUG
9907 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9908 Assert(!fIOWrite);
9909 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9910 Assert(fIOWrite);
9911 else
9912 {
9913 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9914 * statuses, that the VMM device and some others may return. See
9915 * IOM_SUCCESS() for guidance. */
9916 AssertMsg( RT_FAILURE(rcStrict)
9917 || rcStrict == VINF_SUCCESS
9918 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
9919 || rcStrict == VINF_EM_DBG_BREAKPOINT
9920 || rcStrict == VINF_EM_RAW_GUEST_TRAP
9921 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9922 }
9923#endif
9924
9925 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9926 return VBOXSTRICTRC_TODO(rcStrict);
9927}
9928
9929
9930/**
9931 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9932 * VM-exit.
9933 */
9934HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9935{
9936 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9937
9938 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
9939 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9940 AssertRCReturn(rc, rc);
9941 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9942 {
9943 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9944 AssertRCReturn(rc, rc);
9945 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9946 {
9947 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9948
9949 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9950 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9951 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9952 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9953 {
9954 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9955 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9956
9957 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9958 Assert(!pVCpu->hm.s.Event.fPending);
9959 pVCpu->hm.s.Event.fPending = true;
9960 pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9961 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9962 AssertRCReturn(rc, rc);
9963 if (fErrorCodeValid)
9964 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9965 else
9966 pVCpu->hm.s.Event.u32ErrCode = 0;
9967 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9968 && uVector == X86_XCPT_PF)
9969 {
9970 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9971 }
9972
9973 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9974 }
9975 }
9976 }
9977
9978 /** @todo Emulate task switch someday, currently just going back to ring-3 for
9979 * emulation. */
9980 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9981 return VERR_EM_INTERPRETER;
9982}
9983
9984
9985/**
9986 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9987 */
9988HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9989{
9990 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9991 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9992 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9993 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9994 AssertRCReturn(rc, rc);
9995 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9996 return VINF_EM_DBG_STEPPED;
9997}
9998
9999
10000/**
10001 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10002 */
10003HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10004{
10005 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10006
10007 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10008 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10009 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10010 return VINF_SUCCESS;
10011 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10012 return rc;
10013
10014#if 0
10015 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10016 * just sync the whole thing. */
10017 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10018#else
10019 /* Aggressive state sync. for now. */
10020 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10021 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10022 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10023#endif
10024 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10025 AssertRCReturn(rc, rc);
10026
10027 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10028 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10029 switch (uAccessType)
10030 {
10031 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10032 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10033 {
10034 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10035 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10036 {
10037 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10038 }
10039
10040 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10041 GCPhys &= PAGE_BASE_GC_MASK;
10042 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10043 PVM pVM = pVCpu->CTX_SUFF(pVM);
10044 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10045 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10046
10047 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10048 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10049 CPUMCTX2CORE(pMixedCtx), GCPhys);
10050 rc = VBOXSTRICTRC_VAL(rc2);
10051 Log4(("ApicAccess rc=%d\n", rc));
10052 if ( rc == VINF_SUCCESS
10053 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10054 || rc == VERR_PAGE_NOT_PRESENT)
10055 {
10056 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10057 | HM_CHANGED_GUEST_RSP
10058 | HM_CHANGED_GUEST_RFLAGS
10059 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10060 rc = VINF_SUCCESS;
10061 }
10062 break;
10063 }
10064
10065 default:
10066 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10067 rc = VINF_EM_RAW_EMULATE_INSTR;
10068 break;
10069 }
10070
10071 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10072 return rc;
10073}
10074
10075
10076/**
10077 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10078 * VM-exit.
10079 */
10080HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10081{
10082 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10083
10084 /* We should -not- get this VM-exit if the guest's debug registers were active. */
10085 if (pVmxTransient->fWasGuestDebugStateActive)
10086 {
10087 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10088 return VERR_VMX_UNEXPECTED_EXIT_CODE;
10089 }
10090
10091 int rc = VERR_INTERNAL_ERROR_5;
10092 if ( !DBGFIsStepping(pVCpu)
10093 && !pVCpu->hm.s.fSingleInstruction
10094 && !pVmxTransient->fWasHyperDebugStateActive)
10095 {
10096 /* Don't intercept MOV DRx and #DB any more. */
10097 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10098 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10099 AssertRCReturn(rc, rc);
10100
10101 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10102 {
10103#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10104 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10105 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10106 AssertRCReturn(rc, rc);
10107#endif
10108 }
10109
10110 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10111 VMMRZCallRing3Disable(pVCpu);
10112 HM_DISABLE_PREEMPT_IF_NEEDED();
10113
10114 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10115 PVM pVM = pVCpu->CTX_SUFF(pVM);
10116 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10117 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
10118
10119 HM_RESTORE_PREEMPT_IF_NEEDED();
10120 VMMRZCallRing3Enable(pVCpu);
10121
10122#ifdef VBOX_WITH_STATISTICS
10123 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10124 AssertRCReturn(rc, rc);
10125 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10126 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10127 else
10128 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10129#endif
10130 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10131 return VINF_SUCCESS;
10132 }
10133
10134 /*
10135 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10136 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10137 */
10138 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10139 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10140 AssertRCReturn(rc, rc);
10141 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10142
10143 PVM pVM = pVCpu->CTX_SUFF(pVM);
10144 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10145 {
10146 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10147 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10148 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10149 if (RT_SUCCESS(rc))
10150 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10151 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10152 }
10153 else
10154 {
10155 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10156 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10157 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10158 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10159 }
10160
10161 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10162 if (RT_SUCCESS(rc))
10163 {
10164 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10165 AssertRCReturn(rc2, rc2);
10166 }
10167 return rc;
10168}
10169
10170
10171/**
10172 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10173 * Conditional VM-exit.
10174 */
10175HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10176{
10177 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10178 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10179
10180 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10181 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10182 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10183 return VINF_SUCCESS;
10184 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10185 return rc;
10186
10187 RTGCPHYS GCPhys = 0;
10188 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10189
10190#if 0
10191 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10192#else
10193 /* Aggressive state sync. for now. */
10194 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10195 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10196 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10197#endif
10198 AssertRCReturn(rc, rc);
10199
10200 /*
10201 * If we succeed, resume guest execution.
10202 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10203 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10204 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10205 * weird case. See @bugref{6043}.
10206 */
10207 PVM pVM = pVCpu->CTX_SUFF(pVM);
10208 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10209 rc = VBOXSTRICTRC_VAL(rc2);
10210 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10211 if ( rc == VINF_SUCCESS
10212 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10213 || rc == VERR_PAGE_NOT_PRESENT)
10214 {
10215 /* Successfully handled MMIO operation. */
10216 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10217 | HM_CHANGED_GUEST_RSP
10218 | HM_CHANGED_GUEST_RFLAGS
10219 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10220 rc = VINF_SUCCESS;
10221 }
10222 return rc;
10223}
10224
10225
10226/**
10227 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10228 * VM-exit.
10229 */
10230HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10231{
10232 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10233 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10234
10235 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10236 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10237 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10238 return VINF_SUCCESS;
10239 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10240 return rc;
10241
10242 RTGCPHYS GCPhys = 0;
10243 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10244 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10245#if 0
10246 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10247#else
10248 /* Aggressive state sync. for now. */
10249 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10250 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10251 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10252#endif
10253 AssertRCReturn(rc, rc);
10254
10255 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10256 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10257
10258 RTGCUINT uErrorCode = 0;
10259 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10260 uErrorCode |= X86_TRAP_PF_ID;
10261 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10262 uErrorCode |= X86_TRAP_PF_RW;
10263 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10264 uErrorCode |= X86_TRAP_PF_P;
10265
10266 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10267
10268 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10269 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10270
10271 /* Handle the pagefault trap for the nested shadow table. */
10272 PVM pVM = pVCpu->CTX_SUFF(pVM);
10273 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10274 TRPMResetTrap(pVCpu);
10275
10276 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10277 if ( rc == VINF_SUCCESS
10278 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10279 || rc == VERR_PAGE_NOT_PRESENT)
10280 {
10281 /* Successfully synced our nested page tables. */
10282 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10283 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10284 | HM_CHANGED_GUEST_RSP
10285 | HM_CHANGED_GUEST_RFLAGS);
10286 return VINF_SUCCESS;
10287 }
10288
10289 Log4(("EPT return to ring-3 rc=%d\n"));
10290 return rc;
10291}
10292
10293/** @} */
10294
10295/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10296/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10297/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10298
10299/** @name VM-exit exception handlers.
10300 * @{
10301 */
10302
10303/**
10304 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10305 */
10306static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10307{
10308 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10309 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10310
10311 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10312 AssertRCReturn(rc, rc);
10313
10314 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10315 {
10316 /* Old-style FPU error reporting needs some extra work. */
10317 /** @todo don't fall back to the recompiler, but do it manually. */
10318 return VERR_EM_INTERPRETER;
10319 }
10320
10321 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10322 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10323 return rc;
10324}
10325
10326
10327/**
10328 * VM-exit exception handler for #BP (Breakpoint exception).
10329 */
10330static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10331{
10332 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10333 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10334
10335 /** @todo Try optimize this by not saving the entire guest state unless
10336 * really needed. */
10337 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10338 AssertRCReturn(rc, rc);
10339
10340 PVM pVM = pVCpu->CTX_SUFF(pVM);
10341 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10342 if (rc == VINF_EM_RAW_GUEST_TRAP)
10343 {
10344 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10345 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10346 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10347 AssertRCReturn(rc, rc);
10348
10349 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10350 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10351 }
10352
10353 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10354 return rc;
10355}
10356
10357
10358/**
10359 * VM-exit exception handler for #DB (Debug exception).
10360 */
10361static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10362{
10363 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10364 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10365 Log6(("XcptDB\n"));
10366
10367 /*
10368 * Get the DR6-like values from the exit qualification and pass it to DBGF
10369 * for processing.
10370 */
10371 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10372 AssertRCReturn(rc, rc);
10373
10374 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10375 uint64_t uDR6 = X86_DR6_INIT_VAL;
10376 uDR6 |= ( pVmxTransient->uExitQualification
10377 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10378
10379 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10380 if (rc == VINF_EM_RAW_GUEST_TRAP)
10381 {
10382 /*
10383 * The exception was for the guest. Update DR6, DR7.GD and
10384 * IA32_DEBUGCTL.LBR before forwarding it.
10385 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10386 */
10387 VMMRZCallRing3Disable(pVCpu);
10388 HM_DISABLE_PREEMPT_IF_NEEDED();
10389
10390 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10391 pMixedCtx->dr[6] |= uDR6;
10392 if (CPUMIsGuestDebugStateActive(pVCpu))
10393 ASMSetDR6(pMixedCtx->dr[6]);
10394
10395 HM_RESTORE_PREEMPT_IF_NEEDED();
10396 VMMRZCallRing3Enable(pVCpu);
10397
10398 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10399 AssertRCReturn(rc, rc);
10400
10401 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10402 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10403
10404 /* Paranoia. */
10405 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10406 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10407
10408 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10409 AssertRCReturn(rc, rc);
10410
10411 /*
10412 * Raise #DB in the guest.
10413 */
10414 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10415 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10416 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10417 AssertRCReturn(rc, rc);
10418 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10419 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10420 return VINF_SUCCESS;
10421 }
10422
10423 /*
10424 * Not a guest trap, must be a hypervisor related debug event then.
10425 * Update DR6 in case someone is interested in it.
10426 */
10427 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10428 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10429 CPUMSetHyperDR6(pVCpu, uDR6);
10430
10431 return rc;
10432}
10433
10434
10435/**
10436 * VM-exit exception handler for #NM (Device-not-available exception: floating
10437 * point exception).
10438 */
10439static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10440{
10441 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10442
10443 /* We require CR0 and EFER. EFER is always up-to-date. */
10444 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10445 AssertRCReturn(rc, rc);
10446
10447 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10448 VMMRZCallRing3Disable(pVCpu);
10449 HM_DISABLE_PREEMPT_IF_NEEDED();
10450
10451 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10452 if (pVmxTransient->fWasGuestFPUStateActive)
10453 {
10454 rc = VINF_EM_RAW_GUEST_TRAP;
10455 Assert(CPUMIsGuestFPUStateActive(pVCpu) || VMCPU_HMCF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
10456 }
10457 else
10458 {
10459#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10460 Assert(!pVmxTransient->fWasGuestFPUStateActive);
10461#endif
10462 /* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10463 rc = CPUMR0LoadGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10464 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10465 }
10466
10467 HM_RESTORE_PREEMPT_IF_NEEDED();
10468 VMMRZCallRing3Enable(pVCpu);
10469
10470 if (rc == VINF_SUCCESS)
10471 {
10472 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10473 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10474 }
10475 else
10476 {
10477 /* Forward #NM to the guest. */
10478 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10479 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10480 AssertRCReturn(rc, rc);
10481 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10482 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10483 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10484 }
10485
10486 return VINF_SUCCESS;
10487}
10488
10489
10490/**
10491 * VM-exit exception handler for #GP (General-protection exception).
10492 *
10493 * @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
10494 */
10495static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10496{
10497 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10498 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10499
10500 int rc = VERR_INTERNAL_ERROR_5;
10501 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10502 {
10503#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10504 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10505 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10506 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10507 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10508 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10509 AssertRCReturn(rc, rc);
10510 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10511 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10512 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10513 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10514 return rc;
10515#else
10516 /* We don't intercept #GP. */
10517 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10518 return VERR_VMX_UNEXPECTED_EXCEPTION;
10519#endif
10520 }
10521
10522 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10523 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10524
10525 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10526 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10527 AssertRCReturn(rc, rc);
10528
10529 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10530 uint32_t cbOp = 0;
10531 PVM pVM = pVCpu->CTX_SUFF(pVM);
10532 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10533 if (RT_SUCCESS(rc))
10534 {
10535 rc = VINF_SUCCESS;
10536 Assert(cbOp == pDis->cbInstr);
10537 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10538 switch (pDis->pCurInstr->uOpcode)
10539 {
10540 case OP_CLI:
10541 {
10542 pMixedCtx->eflags.Bits.u1IF = 0;
10543 pMixedCtx->rip += pDis->cbInstr;
10544 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
10545 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10546 break;
10547 }
10548
10549 case OP_STI:
10550 {
10551 pMixedCtx->eflags.Bits.u1IF = 1;
10552 pMixedCtx->rip += pDis->cbInstr;
10553 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10554 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10555 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
10556 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10557 break;
10558 }
10559
10560 case OP_HLT:
10561 {
10562 rc = VINF_EM_HALT;
10563 pMixedCtx->rip += pDis->cbInstr;
10564 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10565 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10566 break;
10567 }
10568
10569 case OP_POPF:
10570 {
10571 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10572 uint32_t cbParm = 0;
10573 uint32_t uMask = 0;
10574 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10575 {
10576 cbParm = 4;
10577 uMask = 0xffffffff;
10578 }
10579 else
10580 {
10581 cbParm = 2;
10582 uMask = 0xffff;
10583 }
10584
10585 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10586 RTGCPTR GCPtrStack = 0;
10587 X86EFLAGS Eflags;
10588 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10589 &GCPtrStack);
10590 if (RT_SUCCESS(rc))
10591 {
10592 Assert(sizeof(Eflags.u32) >= cbParm);
10593 Eflags.u32 = 0;
10594 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10595 }
10596 if (RT_FAILURE(rc))
10597 {
10598 rc = VERR_EM_INTERPRETER;
10599 break;
10600 }
10601 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10602 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10603 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10604 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
10605 pMixedCtx->esp += cbParm;
10606 pMixedCtx->esp &= uMask;
10607 pMixedCtx->rip += pDis->cbInstr;
10608
10609 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10610 | HM_CHANGED_GUEST_RSP
10611 | HM_CHANGED_GUEST_RFLAGS);
10612 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10613 break;
10614 }
10615
10616 case OP_PUSHF:
10617 {
10618 uint32_t cbParm = 0;
10619 uint32_t uMask = 0;
10620 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10621 {
10622 cbParm = 4;
10623 uMask = 0xffffffff;
10624 }
10625 else
10626 {
10627 cbParm = 2;
10628 uMask = 0xffff;
10629 }
10630
10631 /* Get the stack pointer & push the contents of eflags onto the stack. */
10632 RTGCPTR GCPtrStack = 0;
10633 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10634 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10635 if (RT_FAILURE(rc))
10636 {
10637 rc = VERR_EM_INTERPRETER;
10638 break;
10639 }
10640 X86EFLAGS Eflags = pMixedCtx->eflags;
10641 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10642 Eflags.Bits.u1RF = 0;
10643 Eflags.Bits.u1VM = 0;
10644
10645 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10646 if (RT_FAILURE(rc))
10647 {
10648 rc = VERR_EM_INTERPRETER;
10649 break;
10650 }
10651 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10652 pMixedCtx->esp -= cbParm;
10653 pMixedCtx->esp &= uMask;
10654 pMixedCtx->rip += pDis->cbInstr;
10655 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
10656 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10657 break;
10658 }
10659
10660 case OP_IRET:
10661 {
10662 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10663 * instruction reference. */
10664 RTGCPTR GCPtrStack = 0;
10665 uint32_t uMask = 0xffff;
10666 uint16_t aIretFrame[3];
10667 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10668 {
10669 rc = VERR_EM_INTERPRETER;
10670 break;
10671 }
10672 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10673 &GCPtrStack);
10674 if (RT_SUCCESS(rc))
10675 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10676 if (RT_FAILURE(rc))
10677 {
10678 rc = VERR_EM_INTERPRETER;
10679 break;
10680 }
10681 pMixedCtx->eip = 0;
10682 pMixedCtx->ip = aIretFrame[0];
10683 pMixedCtx->cs.Sel = aIretFrame[1];
10684 pMixedCtx->cs.ValidSel = aIretFrame[1];
10685 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10686 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10687 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10688 pMixedCtx->sp += sizeof(aIretFrame);
10689 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10690 | HM_CHANGED_GUEST_SEGMENT_REGS
10691 | HM_CHANGED_GUEST_RSP
10692 | HM_CHANGED_GUEST_RFLAGS);
10693 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10694 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10695 break;
10696 }
10697
10698 case OP_INT:
10699 {
10700 uint16_t uVector = pDis->Param1.uValue & 0xff;
10701 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10702 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10703 break;
10704 }
10705
10706 case OP_INTO:
10707 {
10708 if (pMixedCtx->eflags.Bits.u1OF)
10709 {
10710 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10711 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10712 }
10713 break;
10714 }
10715
10716 default:
10717 {
10718 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10719 EMCODETYPE_SUPERVISOR);
10720 rc = VBOXSTRICTRC_VAL(rc2);
10721 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
10722 Log4(("#GP rc=%Rrc\n", rc));
10723 break;
10724 }
10725 }
10726 }
10727 else
10728 rc = VERR_EM_INTERPRETER;
10729
10730 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10731 ("#GP Unexpected rc=%Rrc\n", rc));
10732 return rc;
10733}
10734
10735
10736/**
10737 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10738 * the exception reported in the VMX transient structure back into the VM.
10739 *
10740 * @remarks Requires uExitIntrInfo in the VMX transient structure to be
10741 * up-to-date.
10742 */
10743static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10744{
10745 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10746
10747 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10748 hmR0VmxCheckExitDueToEventDelivery(). */
10749 int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10750 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10751 AssertRCReturn(rc, rc);
10752 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10753
10754 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10755 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10756 return VINF_SUCCESS;
10757}
10758
10759
10760/**
10761 * VM-exit exception handler for #PF (Page-fault exception).
10762 */
10763static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10764{
10765 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10766 PVM pVM = pVCpu->CTX_SUFF(pVM);
10767 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10768 rc |= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10769 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10770 AssertRCReturn(rc, rc);
10771
10772#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
10773 if (pVM->hm.s.fNestedPaging)
10774 {
10775 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10776 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10777 {
10778 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10779 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10780 0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10781 }
10782 else
10783 {
10784 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10785 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10786 Log4(("Pending #DF due to vectoring #PF. NP\n"));
10787 }
10788 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10789 return rc;
10790 }
10791#else
10792 Assert(!pVM->hm.s.fNestedPaging);
10793#endif
10794
10795 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10796 AssertRCReturn(rc, rc);
10797
10798 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10799 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10800
10801 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10802 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10803 (RTGCPTR)pVmxTransient->uExitQualification);
10804
10805 Log4(("#PF: rc=%Rrc\n", rc));
10806 if (rc == VINF_SUCCESS)
10807 {
10808 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10809 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
10810 * memory? We don't update the whole state here... */
10811 VMCPU_HMCF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10812 | HM_CHANGED_GUEST_RSP
10813 | HM_CHANGED_GUEST_RFLAGS
10814 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10815 TRPMResetTrap(pVCpu);
10816 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10817 return rc;
10818 }
10819 else if (rc == VINF_EM_RAW_GUEST_TRAP)
10820 {
10821 if (!pVmxTransient->fVectoringPF)
10822 {
10823 /* It's a guest page fault and needs to be reflected to the guest. */
10824 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10825 TRPMResetTrap(pVCpu);
10826 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10827 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10828 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10829 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10830 }
10831 else
10832 {
10833 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10834 TRPMResetTrap(pVCpu);
10835 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10836 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10837 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10838 }
10839
10840 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10841 return VINF_SUCCESS;
10842 }
10843
10844 TRPMResetTrap(pVCpu);
10845 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10846 return rc;
10847}
10848
10849/** @} */
10850
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette