VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 48569

Last change on this file since 48569 was 48565, checked in by vboxsync, 11 years ago

VMM/HM: Added total VM-exits STAM.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 452.8 KB
Line 
1/* $Id: HMVMXR0.cpp 48565 2013-09-19 22:16:00Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#define HMVMX_ALWAYS_SWAP_FPU_STATE
45#endif
46
47
48/*******************************************************************************
49* Defined Constants And Macros *
50*******************************************************************************/
51#if defined(RT_ARCH_AMD64)
52# define HMVMX_IS_64BIT_HOST_MODE() (true)
53typedef RTHCUINTREG HMVMXHCUINTREG;
54#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
55extern "C" uint32_t g_fVMXIs64bitHost;
56# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
57typedef uint64_t HMVMXHCUINTREG;
58#else
59# define HMVMX_IS_64BIT_HOST_MODE() (false)
60typedef RTHCUINTREG HMVMXHCUINTREG;
61#endif
62
63/** Use the function table. */
64#define HMVMX_USE_FUNCTION_TABLE
65
66/** Determine which tagged-TLB flush handler to use. */
67#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72/** @name Updated-guest-state flags.
73 * @{ */
74#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95 | HMVMX_UPDATED_GUEST_RSP \
96 | HMVMX_UPDATED_GUEST_RFLAGS \
97 | HMVMX_UPDATED_GUEST_CR0 \
98 | HMVMX_UPDATED_GUEST_CR3 \
99 | HMVMX_UPDATED_GUEST_CR4 \
100 | HMVMX_UPDATED_GUEST_GDTR \
101 | HMVMX_UPDATED_GUEST_IDTR \
102 | HMVMX_UPDATED_GUEST_LDTR \
103 | HMVMX_UPDATED_GUEST_TR \
104 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105 | HMVMX_UPDATED_GUEST_DEBUG \
106 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113 | HMVMX_UPDATED_GUEST_APIC_STATE)
114/** @} */
115
116/** @name
117 * Flags to skip redundant reads of some common VMCS fields that are not part of
118 * the guest-CPU state but are in the transient structure.
119 */
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126/** @} */
127
128/** @name
129 * States of the VMCS.
130 *
131 * This does not reflect all possible VMCS states but currently only those
132 * needed for maintaining the VMCS consistently even when thread-context hooks
133 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
134 */
135#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
136#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
137#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
138/** @} */
139
140/**
141 * Exception bitmap mask for real-mode guests (real-on-v86).
142 *
143 * We need to intercept all exceptions manually (except #PF). #NM is also
144 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
145 * even in real-mode if we have Nested Paging support.
146 */
147#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
148 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
149 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
150 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
151 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
153 | RT_BIT(X86_XCPT_XF))
154
155/**
156 * Exception bitmap mask for all contributory exceptions.
157 *
158 * Page fault is deliberately excluded here as it's conditional as to whether
159 * it's contributory or benign. Page faults are handled separately.
160 */
161#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
162 | RT_BIT(X86_XCPT_DE))
163
164/** Maximum VM-instruction error number. */
165#define HMVMX_INSTR_ERROR_MAX 28
166
167/** Profiling macro. */
168#ifdef HM_PROFILE_EXIT_DISPATCH
169# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171#else
172# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174#endif
175
176/** Assert that preemption is disabled or covered by thread-context hooks. */
177#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
178 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180/** Assert that we haven't migrated CPUs when thread-context hooks are not
181 * used. */
182#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184 ("Illegal migration! Entered on CPU %u Current %u\n", \
185 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187/*******************************************************************************
188* Structures and Typedefs *
189*******************************************************************************/
190/**
191 * VMX transient state.
192 *
193 * A state structure for holding miscellaneous information across
194 * VMX non-root operation and restored after the transition.
195 */
196typedef struct VMXTRANSIENT
197{
198 /** The host's rflags/eflags. */
199 RTCCUINTREG uEflags;
200#if HC_ARCH_BITS == 32
201 uint32_t u32Alignment0;
202#endif
203 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
204 uint64_t u64LStarMsr;
205 /** The guest's TPR value used for TPR shadowing. */
206 uint8_t u8GuestTpr;
207 /** Alignment. */
208 uint8_t abAlignment0[7];
209
210 /** The basic VM-exit reason. */
211 uint16_t uExitReason;
212 /** Alignment. */
213 uint16_t u16Alignment0;
214 /** The VM-exit interruption error code. */
215 uint32_t uExitIntrErrorCode;
216 /** The VM-exit exit qualification. */
217 uint64_t uExitQualification;
218
219 /** The VM-exit interruption-information field. */
220 uint32_t uExitIntrInfo;
221 /** The VM-exit instruction-length field. */
222 uint32_t cbInstr;
223 /** The VM-exit instruction-information field. */
224 union
225 {
226 /** Plain unsigned int representation. */
227 uint32_t u;
228 /** INS and OUTS information. */
229 struct
230 {
231 uint32_t u6Reserved0 : 6;
232 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
233 uint32_t u3AddrSize : 3;
234 uint32_t u5Reserved1 : 5;
235 /** The segment register (X86_SREG_XXX). */
236 uint32_t iSegReg : 3;
237 uint32_t uReserved2 : 14;
238 } StrIo;
239 } ExitInstrInfo;
240 /** Whether the VM-entry failed or not. */
241 bool fVMEntryFailed;
242 /** Alignment. */
243 uint8_t abAlignment1[3];
244
245 /** The VM-entry interruption-information field. */
246 uint32_t uEntryIntrInfo;
247 /** The VM-entry exception error code field. */
248 uint32_t uEntryXcptErrorCode;
249 /** The VM-entry instruction length field. */
250 uint32_t cbEntryInstr;
251
252 /** IDT-vectoring information field. */
253 uint32_t uIdtVectoringInfo;
254 /** IDT-vectoring error code. */
255 uint32_t uIdtVectoringErrorCode;
256
257 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
258 uint32_t fVmcsFieldsRead;
259 /** Whether TSC-offsetting should be setup before VM-entry. */
260 bool fUpdateTscOffsettingAndPreemptTimer;
261 /** Whether the VM-exit was caused by a page-fault during delivery of a
262 * contributory exception or a page-fault. */
263 bool fVectoringPF;
264 /** Whether the guest FPU was active at the time of VM-exit. */
265 bool fWasGuestFPUStateActive;
266 /** Whether the guest debug state was active at the time of VM-exit. */
267 bool fWasGuestDebugStateActive;
268 /** Whether the hyper debug state was active at the time of VM-exit. */
269 bool fWasHyperDebugStateActive;
270} VMXTRANSIENT;
271AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
272AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
273AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
274AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
275/** Pointer to VMX transient state. */
276typedef VMXTRANSIENT *PVMXTRANSIENT;
277
278
279/**
280 * MSR-bitmap read permissions.
281 */
282typedef enum VMXMSREXITREAD
283{
284 /** Reading this MSR causes a VM-exit. */
285 VMXMSREXIT_INTERCEPT_READ = 0xb,
286 /** Reading this MSR does not cause a VM-exit. */
287 VMXMSREXIT_PASSTHRU_READ
288} VMXMSREXITREAD;
289
290/**
291 * MSR-bitmap write permissions.
292 */
293typedef enum VMXMSREXITWRITE
294{
295 /** Writing to this MSR causes a VM-exit. */
296 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
297 /** Writing to this MSR does not cause a VM-exit. */
298 VMXMSREXIT_PASSTHRU_WRITE
299} VMXMSREXITWRITE;
300
301/**
302 * VMX VM-exit handler.
303 *
304 * @returns VBox status code.
305 * @param pVCpu Pointer to the VMCPU.
306 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
307 * out-of-sync. Make sure to update the required
308 * fields before using them.
309 * @param pVmxTransient Pointer to the VMX-transient structure.
310 */
311#ifndef HMVMX_USE_FUNCTION_TABLE
312typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313#else
314typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
315/** Pointer to VM-exit handler. */
316typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
317#endif
318
319
320/*******************************************************************************
321* Internal Functions *
322*******************************************************************************/
323static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
324static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
325static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
326static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
327 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
328#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
329static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
330#endif
331#ifndef HMVMX_USE_FUNCTION_TABLE
332DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
333# define HMVMX_EXIT_DECL static int
334#else
335# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
336#endif
337
338/** @name VM-exit handlers.
339 * @{
340 */
341static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
342static FNVMXEXITHANDLER hmR0VmxExitExtInt;
343static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
344static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
345static FNVMXEXITHANDLER hmR0VmxExitSipi;
346static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
347static FNVMXEXITHANDLER hmR0VmxExitSmi;
348static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
349static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
350static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
351static FNVMXEXITHANDLER hmR0VmxExitCpuid;
352static FNVMXEXITHANDLER hmR0VmxExitGetsec;
353static FNVMXEXITHANDLER hmR0VmxExitHlt;
354static FNVMXEXITHANDLER hmR0VmxExitInvd;
355static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
356static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
357static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
358static FNVMXEXITHANDLER hmR0VmxExitRsm;
359static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
360static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
361static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
362static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
363static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
364static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
365static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
366static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
367static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
368static FNVMXEXITHANDLER hmR0VmxExitMwait;
369static FNVMXEXITHANDLER hmR0VmxExitMtf;
370static FNVMXEXITHANDLER hmR0VmxExitMonitor;
371static FNVMXEXITHANDLER hmR0VmxExitPause;
372static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
373static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
374static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
375static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
376static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
377static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
378static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
379static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
380static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
381static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
382static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
383static FNVMXEXITHANDLER hmR0VmxExitRdrand;
384static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
385/** @} */
386
387static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
388static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
389static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
390static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
391static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
392static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
393static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
394static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
395
396/*******************************************************************************
397* Global Variables *
398*******************************************************************************/
399#ifdef HMVMX_USE_FUNCTION_TABLE
400
401/**
402 * VMX_EXIT dispatch table.
403 */
404static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
405{
406 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
407 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
408 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
409 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
410 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
411 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
412 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
413 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
414 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
415 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
416 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
417 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
418 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
419 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
420 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
421 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
422 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
423 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
424 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
425 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
426 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
427 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
428 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
429 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
430 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
431 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
432 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
433 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
434 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
435 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
436 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
437 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
438 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
439 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
440 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
441 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
442 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
443 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
444 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
445 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
446 /* 40 UNDEFINED */ hmR0VmxExitPause,
447 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
448 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
449 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
450 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
451 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
452 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
453 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
454 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
455 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
456 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
457 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
458 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
459 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
460 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
461 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
462 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
463 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
464 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
465 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
466};
467#endif /* HMVMX_USE_FUNCTION_TABLE */
468
469#ifdef VBOX_STRICT
470static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
471{
472 /* 0 */ "(Not Used)",
473 /* 1 */ "VMCALL executed in VMX root operation.",
474 /* 2 */ "VMCLEAR with invalid physical address.",
475 /* 3 */ "VMCLEAR with VMXON pointer.",
476 /* 4 */ "VMLAUNCH with non-clear VMCS.",
477 /* 5 */ "VMRESUME with non-launched VMCS.",
478 /* 6 */ "VMRESUME after VMXOFF",
479 /* 7 */ "VM entry with invalid control fields.",
480 /* 8 */ "VM entry with invalid host state fields.",
481 /* 9 */ "VMPTRLD with invalid physical address.",
482 /* 10 */ "VMPTRLD with VMXON pointer.",
483 /* 11 */ "VMPTRLD with incorrect revision identifier.",
484 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
485 /* 13 */ "VMWRITE to read-only VMCS component.",
486 /* 14 */ "(Not Used)",
487 /* 15 */ "VMXON executed in VMX root operation.",
488 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
489 /* 17 */ "VM entry with non-launched executing VMCS.",
490 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
491 /* 19 */ "VMCALL with non-clear VMCS.",
492 /* 20 */ "VMCALL with invalid VM-exit control fields.",
493 /* 21 */ "(Not Used)",
494 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
495 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
496 /* 24 */ "VMCALL with invalid SMM-monitor features.",
497 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
498 /* 26 */ "VM entry with events blocked by MOV SS.",
499 /* 27 */ "(Not Used)",
500 /* 28 */ "Invalid operand to INVEPT/INVVPID."
501};
502#endif /* VBOX_STRICT */
503
504
505
506/**
507 * Updates the VM's last error record. If there was a VMX instruction error,
508 * reads the error data from the VMCS and updates VCPU's last error record as
509 * well.
510 *
511 * @param pVM Pointer to the VM.
512 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
513 * VERR_VMX_UNABLE_TO_START_VM or
514 * VERR_VMX_INVALID_VMCS_FIELD).
515 * @param rc The error code.
516 */
517static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
518{
519 AssertPtr(pVM);
520 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
521 || rc == VERR_VMX_UNABLE_TO_START_VM)
522 {
523 AssertPtrReturnVoid(pVCpu);
524 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
525 }
526 pVM->hm.s.lLastError = rc;
527}
528
529
530/**
531 * Reads the VM-entry interruption-information field from the VMCS into the VMX
532 * transient structure.
533 *
534 * @returns VBox status code.
535 * @param pVmxTransient Pointer to the VMX transient structure.
536 *
537 * @remarks No-long-jump zone!!!
538 */
539DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
540{
541 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
542 AssertRCReturn(rc, rc);
543 return VINF_SUCCESS;
544}
545
546
547/**
548 * Reads the VM-entry exception error code field from the VMCS into
549 * the VMX transient structure.
550 *
551 * @returns VBox status code.
552 * @param pVmxTransient Pointer to the VMX transient structure.
553 *
554 * @remarks No-long-jump zone!!!
555 */
556DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
557{
558 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
559 AssertRCReturn(rc, rc);
560 return VINF_SUCCESS;
561}
562
563
564/**
565 * Reads the VM-entry exception error code field from the VMCS into
566 * the VMX transient structure.
567 *
568 * @returns VBox status code.
569 * @param pVCpu Pointer to the VMCPU.
570 * @param pVmxTransient Pointer to the VMX transient structure.
571 *
572 * @remarks No-long-jump zone!!!
573 */
574DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
575{
576 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
577 AssertRCReturn(rc, rc);
578 return VINF_SUCCESS;
579}
580
581
582/**
583 * Reads the VM-exit interruption-information field from the VMCS into the VMX
584 * transient structure.
585 *
586 * @returns VBox status code.
587 * @param pVCpu Pointer to the VMCPU.
588 * @param pVmxTransient Pointer to the VMX transient structure.
589 */
590DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
591{
592 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
593 {
594 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
595 AssertRCReturn(rc, rc);
596 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
597 }
598 return VINF_SUCCESS;
599}
600
601
602/**
603 * Reads the VM-exit interruption error code from the VMCS into the VMX
604 * transient structure.
605 *
606 * @returns VBox status code.
607 * @param pVCpu Pointer to the VMCPU.
608 * @param pVmxTransient Pointer to the VMX transient structure.
609 */
610DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
611{
612 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
613 {
614 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
615 AssertRCReturn(rc, rc);
616 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
617 }
618 return VINF_SUCCESS;
619}
620
621
622/**
623 * Reads the VM-exit instruction length field from the VMCS into the VMX
624 * transient structure.
625 *
626 * @returns VBox status code.
627 * @param pVCpu Pointer to the VMCPU.
628 * @param pVmxTransient Pointer to the VMX transient structure.
629 */
630DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
631{
632 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
633 {
634 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
635 AssertRCReturn(rc, rc);
636 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
637 }
638 return VINF_SUCCESS;
639}
640
641
642/**
643 * Reads the VM-exit instruction-information field from the VMCS into
644 * the VMX transient structure.
645 *
646 * @returns VBox status code.
647 * @param pVCpu The cross context per CPU structure.
648 * @param pVmxTransient Pointer to the VMX transient structure.
649 */
650DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
651{
652 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
653 {
654 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
655 AssertRCReturn(rc, rc);
656 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
657 }
658 return VINF_SUCCESS;
659}
660
661
662/**
663 * Reads the exit qualification from the VMCS into the VMX transient structure.
664 *
665 * @returns VBox status code.
666 * @param pVCpu Pointer to the VMCPU.
667 * @param pVmxTransient Pointer to the VMX transient structure.
668 */
669DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
670{
671 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
672 {
673 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
674 AssertRCReturn(rc, rc);
675 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
676 }
677 return VINF_SUCCESS;
678}
679
680
681/**
682 * Reads the IDT-vectoring information field from the VMCS into the VMX
683 * transient structure.
684 *
685 * @returns VBox status code.
686 * @param pVmxTransient Pointer to the VMX transient structure.
687 *
688 * @remarks No-long-jump zone!!!
689 */
690DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
691{
692 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
693 {
694 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
695 AssertRCReturn(rc, rc);
696 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
697 }
698 return VINF_SUCCESS;
699}
700
701
702/**
703 * Reads the IDT-vectoring error code from the VMCS into the VMX
704 * transient structure.
705 *
706 * @returns VBox status code.
707 * @param pVmxTransient Pointer to the VMX transient structure.
708 */
709DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
710{
711 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
712 {
713 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
714 AssertRCReturn(rc, rc);
715 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
716 }
717 return VINF_SUCCESS;
718}
719
720
721/**
722 * Enters VMX root mode operation on the current CPU.
723 *
724 * @returns VBox status code.
725 * @param pVM Pointer to the VM (optional, can be NULL, after
726 * a resume).
727 * @param HCPhysCpuPage Physical address of the VMXON region.
728 * @param pvCpuPage Pointer to the VMXON region.
729 */
730static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
731{
732 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
733 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
735
736 if (pVM)
737 {
738 /* Write the VMCS revision dword to the VMXON region. */
739 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
740 }
741
742 /* Enable the VMX bit in CR4 if necessary. */
743 RTCCUINTREG uCr4 = ASMGetCR4();
744 if (!(uCr4 & X86_CR4_VMXE))
745 ASMSetCR4(uCr4 | X86_CR4_VMXE);
746
747 /* Enter VMX root mode. */
748 int rc = VMXEnable(HCPhysCpuPage);
749 if (RT_FAILURE(rc))
750 ASMSetCR4(uCr4);
751
752 return rc;
753}
754
755
756/**
757 * Exits VMX root mode operation on the current CPU.
758 *
759 * @returns VBox status code.
760 */
761static int hmR0VmxLeaveRootMode(void)
762{
763 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
764
765 /* If we're for some reason not in VMX root mode, then don't leave it. */
766 RTCCUINTREG uHostCR4 = ASMGetCR4();
767 if (uHostCR4 & X86_CR4_VMXE)
768 {
769 /* Exit VMX root mode and clear the VMX bit in CR4. */
770 VMXDisable();
771 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
772 return VINF_SUCCESS;
773 }
774
775 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
776}
777
778
779/**
780 * Allocates and maps one physically contiguous page. The allocated page is
781 * zero'd out. (Used by various VT-x structures).
782 *
783 * @returns IPRT status code.
784 * @param pMemObj Pointer to the ring-0 memory object.
785 * @param ppVirt Where to store the virtual address of the
786 * allocation.
787 * @param pPhys Where to store the physical address of the
788 * allocation.
789 */
790DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
791{
792 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
793 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
794 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
795
796 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
797 if (RT_FAILURE(rc))
798 return rc;
799 *ppVirt = RTR0MemObjAddress(*pMemObj);
800 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
801 ASMMemZero32(*ppVirt, PAGE_SIZE);
802 return VINF_SUCCESS;
803}
804
805
806/**
807 * Frees and unmaps an allocated physical page.
808 *
809 * @param pMemObj Pointer to the ring-0 memory object.
810 * @param ppVirt Where to re-initialize the virtual address of
811 * allocation as 0.
812 * @param pHCPhys Where to re-initialize the physical address of the
813 * allocation as 0.
814 */
815DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
816{
817 AssertPtr(pMemObj);
818 AssertPtr(ppVirt);
819 AssertPtr(pHCPhys);
820 if (*pMemObj != NIL_RTR0MEMOBJ)
821 {
822 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
823 AssertRC(rc);
824 *pMemObj = NIL_RTR0MEMOBJ;
825 *ppVirt = 0;
826 *pHCPhys = 0;
827 }
828}
829
830
831/**
832 * Worker function to free VT-x related structures.
833 *
834 * @returns IPRT status code.
835 * @param pVM Pointer to the VM.
836 */
837static void hmR0VmxStructsFree(PVM pVM)
838{
839 for (VMCPUID i = 0; i < pVM->cCpus; i++)
840 {
841 PVMCPU pVCpu = &pVM->aCpus[i];
842 AssertPtr(pVCpu);
843
844#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
845 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
846 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
847#endif
848
849 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
850 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
851
852 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
853 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
854 }
855
856 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
857#ifdef VBOX_WITH_CRASHDUMP_MAGIC
858 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
859#endif
860}
861
862
863/**
864 * Worker function to allocate VT-x related VM structures.
865 *
866 * @returns IPRT status code.
867 * @param pVM Pointer to the VM.
868 */
869static int hmR0VmxStructsAlloc(PVM pVM)
870{
871 /*
872 * Initialize members up-front so we can cleanup properly on allocation failure.
873 */
874#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
875 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
876 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
877 pVM->hm.s.vmx.HCPhys##a_Name = 0;
878
879#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
880 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
881 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
882 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
883
884#ifdef VBOX_WITH_CRASHDUMP_MAGIC
885 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
886#endif
887 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
888
889 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
890 for (VMCPUID i = 0; i < pVM->cCpus; i++)
891 {
892 PVMCPU pVCpu = &pVM->aCpus[i];
893 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
894 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
895 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
896#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
897 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
898 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
899#endif
900 }
901#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
902#undef VMXLOCAL_INIT_VM_MEMOBJ
903
904 /*
905 * Allocate all the VT-x structures.
906 */
907 int rc = VINF_SUCCESS;
908#ifdef VBOX_WITH_CRASHDUMP_MAGIC
909 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
910 if (RT_FAILURE(rc))
911 goto cleanup;
912 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
913 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
914#endif
915
916 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
917 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
918 {
919 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
920 &pVM->hm.s.vmx.HCPhysApicAccess);
921 if (RT_FAILURE(rc))
922 goto cleanup;
923 }
924
925 /*
926 * Initialize per-VCPU VT-x structures.
927 */
928 for (VMCPUID i = 0; i < pVM->cCpus; i++)
929 {
930 PVMCPU pVCpu = &pVM->aCpus[i];
931 AssertPtr(pVCpu);
932
933 /* Allocate the VM control structure (VMCS). */
934 AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
935 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
936 if (RT_FAILURE(rc))
937 goto cleanup;
938
939 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
940 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
941 {
942 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
943 &pVCpu->hm.s.vmx.HCPhysVirtApic);
944 if (RT_FAILURE(rc))
945 goto cleanup;
946 }
947
948 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
949 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
950 {
951 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
952 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
953 if (RT_FAILURE(rc))
954 goto cleanup;
955 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
956 }
957
958#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
959 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
960 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
961 if (RT_FAILURE(rc))
962 goto cleanup;
963
964 /* Allocate the VM-exit MSR-load page for the host MSRs. */
965 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
966 if (RT_FAILURE(rc))
967 goto cleanup;
968#endif
969 }
970
971 return VINF_SUCCESS;
972
973cleanup:
974 hmR0VmxStructsFree(pVM);
975 return rc;
976}
977
978
979/**
980 * Does global VT-x initialization (called during module initialization).
981 *
982 * @returns VBox status code.
983 */
984VMMR0DECL(int) VMXR0GlobalInit(void)
985{
986#ifdef HMVMX_USE_FUNCTION_TABLE
987 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
988# ifdef VBOX_STRICT
989 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
990 Assert(g_apfnVMExitHandlers[i]);
991# endif
992#endif
993 return VINF_SUCCESS;
994}
995
996
997/**
998 * Does global VT-x termination (called during module termination).
999 */
1000VMMR0DECL(void) VMXR0GlobalTerm()
1001{
1002 /* Nothing to do currently. */
1003}
1004
1005
1006/**
1007 * Sets up and activates VT-x on the current CPU.
1008 *
1009 * @returns VBox status code.
1010 * @param pCpu Pointer to the global CPU info struct.
1011 * @param pVM Pointer to the VM (can be NULL after a host resume
1012 * operation).
1013 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1014 * fEnabledByHost is true).
1015 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1016 * @a fEnabledByHost is true).
1017 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1018 * enable VT-x on the host.
1019 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1020 */
1021VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1022 void *pvMsrs)
1023{
1024 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1025 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1026 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1027
1028 /* Enable VT-x if it's not already enabled by the host. */
1029 if (!fEnabledByHost)
1030 {
1031 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1032 if (RT_FAILURE(rc))
1033 return rc;
1034 }
1035
1036 /*
1037 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1038 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1039 */
1040 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1041 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1042 {
1043 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1044 pCpu->fFlushAsidBeforeUse = false;
1045 }
1046 else
1047 pCpu->fFlushAsidBeforeUse = true;
1048
1049 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1050 ++pCpu->cTlbFlushes;
1051
1052 return VINF_SUCCESS;
1053}
1054
1055
1056/**
1057 * Deactivates VT-x on the current CPU.
1058 *
1059 * @returns VBox status code.
1060 * @param pCpu Pointer to the global CPU info struct.
1061 * @param pvCpuPage Pointer to the VMXON region.
1062 * @param HCPhysCpuPage Physical address of the VMXON region.
1063 *
1064 * @remarks This function should never be called when SUPR0EnableVTx() or
1065 * similar was used to enable VT-x on the host.
1066 */
1067VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1068{
1069 NOREF(pCpu);
1070 NOREF(pvCpuPage);
1071 NOREF(HCPhysCpuPage);
1072
1073 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1074 return hmR0VmxLeaveRootMode();
1075}
1076
1077
1078/**
1079 * Sets the permission bits for the specified MSR in the MSR bitmap.
1080 *
1081 * @param pVCpu Pointer to the VMCPU.
1082 * @param uMSR The MSR value.
1083 * @param enmRead Whether reading this MSR causes a VM-exit.
1084 * @param enmWrite Whether writing this MSR causes a VM-exit.
1085 */
1086static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1087{
1088 int32_t iBit;
1089 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1090
1091 /*
1092 * Layout:
1093 * 0x000 - 0x3ff - Low MSR read bits
1094 * 0x400 - 0x7ff - High MSR read bits
1095 * 0x800 - 0xbff - Low MSR write bits
1096 * 0xc00 - 0xfff - High MSR write bits
1097 */
1098 if (uMsr <= 0x00001FFF)
1099 iBit = uMsr;
1100 else if ( uMsr >= 0xC0000000
1101 && uMsr <= 0xC0001FFF)
1102 {
1103 iBit = (uMsr - 0xC0000000);
1104 pbMsrBitmap += 0x400;
1105 }
1106 else
1107 {
1108 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1109 return;
1110 }
1111
1112 Assert(iBit <= 0x1fff);
1113 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1114 ASMBitSet(pbMsrBitmap, iBit);
1115 else
1116 ASMBitClear(pbMsrBitmap, iBit);
1117
1118 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1119 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1120 else
1121 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1122}
1123
1124
1125/**
1126 * Flushes the TLB using EPT.
1127 *
1128 * @returns VBox status code.
1129 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1130 * enmFlush).
1131 * @param enmFlush Type of flush.
1132 *
1133 * @remarks Caller is responsible for making sure this function is called only
1134 * when NestedPaging is supported and providing @a enmFlush that is
1135 * supported by the CPU.
1136 */
1137static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1138{
1139 uint64_t au64Descriptor[2];
1140 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1141 au64Descriptor[0] = 0;
1142 else
1143 {
1144 Assert(pVCpu);
1145 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1146 }
1147 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1148
1149 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1150 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1151 rc));
1152 if ( RT_SUCCESS(rc)
1153 && pVCpu)
1154 {
1155 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1156 }
1157}
1158
1159
1160/**
1161 * Flushes the TLB using VPID.
1162 *
1163 * @returns VBox status code.
1164 * @param pVM Pointer to the VM.
1165 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1166 * enmFlush).
1167 * @param enmFlush Type of flush.
1168 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1169 * on @a enmFlush).
1170 */
1171static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1172{
1173 AssertPtr(pVM);
1174 Assert(pVM->hm.s.vmx.fVpid);
1175
1176 uint64_t au64Descriptor[2];
1177 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1178 {
1179 au64Descriptor[0] = 0;
1180 au64Descriptor[1] = 0;
1181 }
1182 else
1183 {
1184 AssertPtr(pVCpu);
1185 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1186 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1187 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1188 au64Descriptor[1] = GCPtr;
1189 }
1190
1191 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1192 AssertMsg(rc == VINF_SUCCESS,
1193 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1194 if ( RT_SUCCESS(rc)
1195 && pVCpu)
1196 {
1197 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1198 }
1199}
1200
1201
1202/**
1203 * Invalidates a guest page by guest virtual address. Only relevant for
1204 * EPT/VPID, otherwise there is nothing really to invalidate.
1205 *
1206 * @returns VBox status code.
1207 * @param pVM Pointer to the VM.
1208 * @param pVCpu Pointer to the VMCPU.
1209 * @param GCVirt Guest virtual address of the page to invalidate.
1210 */
1211VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1212{
1213 AssertPtr(pVM);
1214 AssertPtr(pVCpu);
1215 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1216
1217 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1218 if (!fFlushPending)
1219 {
1220 /*
1221 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1222 * See @bugref{6043} and @bugref{6177}.
1223 *
1224 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1225 * function maybe called in a loop with individual addresses.
1226 */
1227 if (pVM->hm.s.vmx.fVpid)
1228 {
1229 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1230 {
1231 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1232 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1233 }
1234 else
1235 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1236 }
1237 else if (pVM->hm.s.fNestedPaging)
1238 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1239 }
1240
1241 return VINF_SUCCESS;
1242}
1243
1244
1245/**
1246 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1247 * otherwise there is nothing really to invalidate.
1248 *
1249 * @returns VBox status code.
1250 * @param pVM Pointer to the VM.
1251 * @param pVCpu Pointer to the VMCPU.
1252 * @param GCPhys Guest physical address of the page to invalidate.
1253 */
1254VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1255{
1256 LogFlowFunc(("%RGp\n", GCPhys));
1257
1258 /*
1259 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1260 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1261 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1262 */
1263 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1264 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1265 return VINF_SUCCESS;
1266}
1267
1268
1269/**
1270 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1271 * case where neither EPT nor VPID is supported by the CPU.
1272 *
1273 * @param pVM Pointer to the VM.
1274 * @param pVCpu Pointer to the VMCPU.
1275 * @param pCpu Pointer to the global HM struct.
1276 *
1277 * @remarks Called with interrupts disabled.
1278 */
1279static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1280{
1281 AssertPtr(pVCpu);
1282 AssertPtr(pCpu);
1283 NOREF(pVM);
1284
1285 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1286 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1287
1288 pVCpu->hm.s.TlbShootdown.cPages = 0;
1289 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1290 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1291 pVCpu->hm.s.fForceTLBFlush = false;
1292 return;
1293}
1294
1295
1296/**
1297 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1298 *
1299 * @param pVM Pointer to the VM.
1300 * @param pVCpu Pointer to the VMCPU.
1301 * @param pCpu Pointer to the global HM CPU struct.
1302 * @remarks All references to "ASID" in this function pertains to "VPID" in
1303 * Intel's nomenclature. The reason is, to avoid confusion in compare
1304 * statements since the host-CPU copies are named "ASID".
1305 *
1306 * @remarks Called with interrupts disabled.
1307 */
1308static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1309{
1310#ifdef VBOX_WITH_STATISTICS
1311 bool fTlbFlushed = false;
1312# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1313# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1314 if (!fTlbFlushed) \
1315 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1316 } while (0)
1317#else
1318# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1319# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1320#endif
1321
1322 AssertPtr(pVM);
1323 AssertPtr(pCpu);
1324 AssertPtr(pVCpu);
1325 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1326 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1327 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1328
1329
1330 /*
1331 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1332 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1333 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1334 */
1335 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1336 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1337 {
1338 ++pCpu->uCurrentAsid;
1339 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1340 {
1341 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1342 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1343 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1344 }
1345
1346 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1347 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1348 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1349
1350 /*
1351 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1352 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1353 */
1354 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1355 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1356 HMVMX_SET_TAGGED_TLB_FLUSHED();
1357 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1358 }
1359
1360 /* Check for explicit TLB shootdowns. */
1361 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1362 {
1363 /*
1364 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1365 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1366 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1367 * but not guest-physical mappings.
1368 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1369 */
1370 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1371 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1372 HMVMX_SET_TAGGED_TLB_FLUSHED();
1373 }
1374
1375 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1376 * not be executed. See hmQueueInvlPage() where it is commented
1377 * out. Support individual entry flushing someday. */
1378 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1379 {
1380 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1381
1382 /*
1383 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1384 * as supported by the CPU.
1385 */
1386 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1387 {
1388 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1389 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1390 }
1391 else
1392 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1393
1394 HMVMX_SET_TAGGED_TLB_FLUSHED();
1395 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1396 }
1397
1398 pVCpu->hm.s.TlbShootdown.cPages = 0;
1399 pVCpu->hm.s.fForceTLBFlush = false;
1400
1401 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1402
1403 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1404 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1405 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1406 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1407 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1408 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1409 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1410 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1411
1412 /* Update VMCS with the VPID. */
1413 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1414 AssertRC(rc);
1415
1416#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1417}
1418
1419
1420/**
1421 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1422 *
1423 * @returns VBox status code.
1424 * @param pVM Pointer to the VM.
1425 * @param pVCpu Pointer to the VMCPU.
1426 * @param pCpu Pointer to the global HM CPU struct.
1427 *
1428 * @remarks Called with interrupts disabled.
1429 */
1430static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1431{
1432 AssertPtr(pVM);
1433 AssertPtr(pVCpu);
1434 AssertPtr(pCpu);
1435 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1436 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1437
1438 /*
1439 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1440 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1441 */
1442 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1443 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1444 {
1445 pVCpu->hm.s.fForceTLBFlush = true;
1446 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1447 }
1448
1449 /* Check for explicit TLB shootdown flushes. */
1450 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1451 {
1452 pVCpu->hm.s.fForceTLBFlush = true;
1453 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1454 }
1455
1456 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1457 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1458
1459 if (pVCpu->hm.s.fForceTLBFlush)
1460 {
1461 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1462 pVCpu->hm.s.fForceTLBFlush = false;
1463 }
1464 else
1465 {
1466 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1467 * not be executed. See hmQueueInvlPage() where it is commented
1468 * out. Support individual entry flushing someday. */
1469 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1470 {
1471 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1472 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1473 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1474 }
1475 else
1476 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1477 }
1478
1479 pVCpu->hm.s.TlbShootdown.cPages = 0;
1480 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1481}
1482
1483
1484/**
1485 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1486 *
1487 * @returns VBox status code.
1488 * @param pVM Pointer to the VM.
1489 * @param pVCpu Pointer to the VMCPU.
1490 * @param pCpu Pointer to the global HM CPU struct.
1491 *
1492 * @remarks Called with interrupts disabled.
1493 */
1494static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1495{
1496 AssertPtr(pVM);
1497 AssertPtr(pVCpu);
1498 AssertPtr(pCpu);
1499 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1500 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1501
1502 /*
1503 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1504 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1505 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1506 */
1507 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1508 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1509 {
1510 pVCpu->hm.s.fForceTLBFlush = true;
1511 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1512 }
1513
1514 /* Check for explicit TLB shootdown flushes. */
1515 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1516 {
1517 /*
1518 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1519 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1520 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1521 */
1522 pVCpu->hm.s.fForceTLBFlush = true;
1523 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1524 }
1525
1526 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1527 if (pVCpu->hm.s.fForceTLBFlush)
1528 {
1529 ++pCpu->uCurrentAsid;
1530 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1531 {
1532 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1533 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1534 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1535 }
1536
1537 pVCpu->hm.s.fForceTLBFlush = false;
1538 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1539 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1540 if (pCpu->fFlushAsidBeforeUse)
1541 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1542 }
1543 else
1544 {
1545 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1546 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1547 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1548 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1549
1550 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1551 * not be executed. See hmQueueInvlPage() where it is commented
1552 * out. Support individual entry flushing someday. */
1553 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1554 {
1555 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1556 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1557 {
1558 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1559 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1560 }
1561 else
1562 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1563 }
1564 else
1565 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1566 }
1567
1568 pVCpu->hm.s.TlbShootdown.cPages = 0;
1569 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1570
1571 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1572 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1573 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1574 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1575 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1576 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1577
1578 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1579 AssertRC(rc);
1580}
1581
1582
1583/**
1584 * Flushes the guest TLB entry based on CPU capabilities.
1585 *
1586 * @param pVCpu Pointer to the VMCPU.
1587 * @param pCpu Pointer to the global HM CPU struct.
1588 */
1589DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1590{
1591 PVM pVM = pVCpu->CTX_SUFF(pVM);
1592 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1593 {
1594 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1595 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1596 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1597 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1598 default:
1599 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1600 break;
1601 }
1602}
1603
1604
1605/**
1606 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1607 * TLB entries from the host TLB before VM-entry.
1608 *
1609 * @returns VBox status code.
1610 * @param pVM Pointer to the VM.
1611 */
1612static int hmR0VmxSetupTaggedTlb(PVM pVM)
1613{
1614 /*
1615 * Determine optimal flush type for Nested Paging.
1616 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1617 * guest execution (see hmR3InitFinalizeR0()).
1618 */
1619 if (pVM->hm.s.fNestedPaging)
1620 {
1621 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1622 {
1623 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1624 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1625 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1626 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1627 else
1628 {
1629 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1630 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1631 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1632 }
1633
1634 /* Make sure the write-back cacheable memory type for EPT is supported. */
1635 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1636 {
1637 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1638 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1639 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1640 }
1641 }
1642 else
1643 {
1644 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1645 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1646 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1647 }
1648 }
1649
1650 /*
1651 * Determine optimal flush type for VPID.
1652 */
1653 if (pVM->hm.s.vmx.fVpid)
1654 {
1655 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1656 {
1657 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1658 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1659 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1660 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1661 else
1662 {
1663 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1664 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1665 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1666 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1667 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1668 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1669 pVM->hm.s.vmx.fVpid = false;
1670 }
1671 }
1672 else
1673 {
1674 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1675 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1676 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1677 pVM->hm.s.vmx.fVpid = false;
1678 }
1679 }
1680
1681 /*
1682 * Setup the handler for flushing tagged-TLBs.
1683 */
1684 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1685 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1686 else if (pVM->hm.s.fNestedPaging)
1687 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1688 else if (pVM->hm.s.vmx.fVpid)
1689 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1690 else
1691 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1692 return VINF_SUCCESS;
1693}
1694
1695
1696/**
1697 * Sets up pin-based VM-execution controls in the VMCS.
1698 *
1699 * @returns VBox status code.
1700 * @param pVM Pointer to the VM.
1701 * @param pVCpu Pointer to the VMCPU.
1702 */
1703static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1704{
1705 AssertPtr(pVM);
1706 AssertPtr(pVCpu);
1707
1708 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1709 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1710
1711 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1712 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1713 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1714
1715 /* Enable the VMX preemption timer. */
1716 if (pVM->hm.s.vmx.fUsePreemptTimer)
1717 {
1718 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1719 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1720 }
1721
1722 if ((val & zap) != val)
1723 {
1724 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1725 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1726 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1727 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1728 }
1729
1730 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1731 AssertRCReturn(rc, rc);
1732
1733 /* Update VCPU with the currently set pin-based VM-execution controls. */
1734 pVCpu->hm.s.vmx.u32PinCtls = val;
1735 return rc;
1736}
1737
1738
1739/**
1740 * Sets up processor-based VM-execution controls in the VMCS.
1741 *
1742 * @returns VBox status code.
1743 * @param pVM Pointer to the VM.
1744 * @param pVMCPU Pointer to the VMCPU.
1745 */
1746static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1747{
1748 AssertPtr(pVM);
1749 AssertPtr(pVCpu);
1750
1751 int rc = VERR_INTERNAL_ERROR_5;
1752 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1753 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1754
1755 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1756 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1757 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1758 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1759 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1760 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1761 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1762
1763 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1764 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1765 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1766 {
1767 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1768 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1769 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1770 }
1771
1772 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1773 if (!pVM->hm.s.fNestedPaging)
1774 {
1775 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1776 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1777 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1778 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1779 }
1780
1781 /* Use TPR shadowing if supported by the CPU. */
1782 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1783 {
1784 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1785 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1786 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1787 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1788 AssertRCReturn(rc, rc);
1789
1790 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1791 /* CR8 writes causes a VM-exit based on TPR threshold. */
1792 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1793 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1794 }
1795 else
1796 {
1797 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1798 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1799 }
1800
1801 /* Use MSR-bitmaps if supported by the CPU. */
1802 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1803 {
1804 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1805
1806 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1807 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1808 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1809 AssertRCReturn(rc, rc);
1810
1811 /*
1812 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1813 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1814 */
1815 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1816 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1817 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1818 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1819 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1820 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1821 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1822 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1823 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1824 }
1825
1826 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1827 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1828 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1829
1830 if ((val & zap) != val)
1831 {
1832 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1833 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1834 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1835 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1836 }
1837
1838 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1839 AssertRCReturn(rc, rc);
1840
1841 /* Update VCPU with the currently set processor-based VM-execution controls. */
1842 pVCpu->hm.s.vmx.u32ProcCtls = val;
1843
1844 /*
1845 * Secondary processor-based VM-execution controls.
1846 */
1847 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1848 {
1849 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1850 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1851
1852 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1853 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1854
1855 if (pVM->hm.s.fNestedPaging)
1856 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1857 else
1858 {
1859 /*
1860 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1861 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1862 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1863 */
1864 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1865 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1866 }
1867
1868 if (pVM->hm.s.vmx.fVpid)
1869 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1870
1871 if (pVM->hm.s.vmx.fUnrestrictedGuest)
1872 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1873
1874 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1875 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1876 * done dynamically. */
1877 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1878 {
1879 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1880 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1881 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1882 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1883 AssertRCReturn(rc, rc);
1884 }
1885
1886 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1887 {
1888 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1889 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1890 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1891 }
1892
1893 if ((val & zap) != val)
1894 {
1895 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1896 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1897 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1898 }
1899
1900 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1901 AssertRCReturn(rc, rc);
1902
1903 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1904 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1905 }
1906 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1907 {
1908 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1909 "available\n"));
1910 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1911 }
1912
1913 return VINF_SUCCESS;
1914}
1915
1916
1917/**
1918 * Sets up miscellaneous (everything other than Pin & Processor-based
1919 * VM-execution) control fields in the VMCS.
1920 *
1921 * @returns VBox status code.
1922 * @param pVM Pointer to the VM.
1923 * @param pVCpu Pointer to the VMCPU.
1924 */
1925static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1926{
1927 AssertPtr(pVM);
1928 AssertPtr(pVCpu);
1929
1930 int rc = VERR_GENERAL_FAILURE;
1931
1932 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1933#if 0
1934 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
1935 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1936 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1937
1938 /*
1939 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1940 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1941 * We thus use the exception bitmap to control it rather than use both.
1942 */
1943 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1944 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1945
1946 /** @todo Explore possibility of using IO-bitmaps. */
1947 /* All IO & IOIO instructions cause VM-exits. */
1948 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1949 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1950
1951 /* Initialize the MSR-bitmap area. */
1952 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1953 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1954 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1955#endif
1956
1957#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1958 /* Setup MSR autoloading/storing. */
1959 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1960 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1961 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1962 AssertRCReturn(rc, rc);
1963 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1964 AssertRCReturn(rc, rc);
1965
1966 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1967 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1968 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1969 AssertRCReturn(rc, rc);
1970#endif
1971
1972 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1973 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1974 AssertRCReturn(rc, rc);
1975
1976 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1977#if 0
1978 /* Setup debug controls */
1979 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1980 AssertRCReturn(rc, rc);
1981 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1982 AssertRCReturn(rc, rc);
1983#endif
1984
1985 return rc;
1986}
1987
1988
1989/**
1990 * Sets up the initial exception bitmap in the VMCS based on static conditions
1991 * (i.e. conditions that cannot ever change after starting the VM).
1992 *
1993 * @returns VBox status code.
1994 * @param pVM Pointer to the VM.
1995 * @param pVCpu Pointer to the VMCPU.
1996 */
1997static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1998{
1999 AssertPtr(pVM);
2000 AssertPtr(pVCpu);
2001
2002 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2003
2004 uint32_t u32XcptBitmap = 0;
2005
2006 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2007 if (!pVM->hm.s.fNestedPaging)
2008 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2009
2010 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2011 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2012 AssertRCReturn(rc, rc);
2013 return rc;
2014}
2015
2016
2017/**
2018 * Sets up the initial guest-state mask. The guest-state mask is consulted
2019 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2020 * for the nested virtualization case (as it would cause a VM-exit).
2021 *
2022 * @param pVCpu Pointer to the VMCPU.
2023 */
2024static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2025{
2026 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2027 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2028 return VINF_SUCCESS;
2029}
2030
2031
2032/**
2033 * Does per-VM VT-x initialization.
2034 *
2035 * @returns VBox status code.
2036 * @param pVM Pointer to the VM.
2037 */
2038VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2039{
2040 LogFlowFunc(("pVM=%p\n", pVM));
2041
2042 int rc = hmR0VmxStructsAlloc(pVM);
2043 if (RT_FAILURE(rc))
2044 {
2045 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2046 return rc;
2047 }
2048
2049 return VINF_SUCCESS;
2050}
2051
2052
2053/**
2054 * Does per-VM VT-x termination.
2055 *
2056 * @returns VBox status code.
2057 * @param pVM Pointer to the VM.
2058 */
2059VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2060{
2061 LogFlowFunc(("pVM=%p\n", pVM));
2062
2063#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2064 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2065 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2066#endif
2067 hmR0VmxStructsFree(pVM);
2068 return VINF_SUCCESS;
2069}
2070
2071
2072/**
2073 * Sets up the VM for execution under VT-x.
2074 * This function is only called once per-VM during initialization.
2075 *
2076 * @returns VBox status code.
2077 * @param pVM Pointer to the VM.
2078 */
2079VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2080{
2081 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2082 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2083
2084 LogFlowFunc(("pVM=%p\n", pVM));
2085
2086 /*
2087 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2088 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2089 */
2090 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2091 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2092 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2093 || !pVM->hm.s.vmx.pRealModeTSS))
2094 {
2095 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2096 return VERR_INTERNAL_ERROR;
2097 }
2098
2099#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2100 /*
2101 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2102 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2103 */
2104 if ( pVM->hm.s.fAllow64BitGuests
2105 && !HMVMX_IS_64BIT_HOST_MODE())
2106 {
2107 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2108 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2109 }
2110#endif
2111
2112 /* Initialize these always, see hmR3InitFinalizeR0().*/
2113 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2114 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2115
2116 /* Setup the tagged-TLB flush handlers. */
2117 int rc = hmR0VmxSetupTaggedTlb(pVM);
2118 if (RT_FAILURE(rc))
2119 {
2120 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2121 return rc;
2122 }
2123
2124 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2125 {
2126 PVMCPU pVCpu = &pVM->aCpus[i];
2127 AssertPtr(pVCpu);
2128 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2129
2130 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2131 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2132
2133 /* Set revision dword at the beginning of the VMCS structure. */
2134 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2135
2136 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2137 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2138 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2139 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2140
2141 /* Load this VMCS as the current VMCS. */
2142 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2143 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2144 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2145
2146 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2147 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2148 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2149
2150 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2151 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2152 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2153
2154 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2155 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2156 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2157
2158 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2159 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2160 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2161
2162 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2163 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2164 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2165
2166#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2167 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2168 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2169 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2170#endif
2171
2172 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2173 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2174 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2175 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2176
2177 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2178
2179 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2180 }
2181
2182 return VINF_SUCCESS;
2183}
2184
2185
2186/**
2187 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2188 * the VMCS.
2189 *
2190 * @returns VBox status code.
2191 * @param pVM Pointer to the VM.
2192 * @param pVCpu Pointer to the VMCPU.
2193 */
2194DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2195{
2196 RTCCUINTREG uReg = ASMGetCR0();
2197 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2198 AssertRCReturn(rc, rc);
2199
2200#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2201 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2202 if (HMVMX_IS_64BIT_HOST_MODE())
2203 {
2204 uint64_t uRegCR3 = HMR0Get64bitCR3();
2205 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2206 }
2207 else
2208#endif
2209 {
2210 uReg = ASMGetCR3();
2211 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2212 }
2213 AssertRCReturn(rc, rc);
2214
2215 uReg = ASMGetCR4();
2216 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2217 AssertRCReturn(rc, rc);
2218 return rc;
2219}
2220
2221
2222/**
2223 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2224 * the host-state area in the VMCS.
2225 *
2226 * @returns VBox status code.
2227 * @param pVM Pointer to the VM.
2228 * @param pVCpu Pointer to the VMCPU.
2229 */
2230DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2231{
2232 int rc = VERR_INTERNAL_ERROR_5;
2233
2234 /*
2235 * Host DS, ES, FS and GS segment registers.
2236 */
2237#if HC_ARCH_BITS == 64
2238 RTSEL uSelDS = ASMGetDS();
2239 RTSEL uSelES = ASMGetES();
2240 RTSEL uSelFS = ASMGetFS();
2241 RTSEL uSelGS = ASMGetGS();
2242#else
2243 RTSEL uSelDS = 0;
2244 RTSEL uSelES = 0;
2245 RTSEL uSelFS = 0;
2246 RTSEL uSelGS = 0;
2247#endif
2248
2249 /* Recalculate which host-state bits need to be manually restored. */
2250 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2251
2252 /*
2253 * Host CS and SS segment registers.
2254 */
2255#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2256 RTSEL uSelCS;
2257 RTSEL uSelSS;
2258 if (HMVMX_IS_64BIT_HOST_MODE())
2259 {
2260 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2261 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2262 }
2263 else
2264 {
2265 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2266 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2267 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2268 }
2269#else
2270 RTSEL uSelCS = ASMGetCS();
2271 RTSEL uSelSS = ASMGetSS();
2272#endif
2273
2274 /*
2275 * Host TR segment register.
2276 */
2277 RTSEL uSelTR = ASMGetTR();
2278
2279#if HC_ARCH_BITS == 64
2280 /*
2281 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2282 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2283 */
2284 if (uSelDS & (X86_SEL_RPL | X86_SEL_LDT))
2285 {
2286 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_DS;
2287 pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2288 uSelDS = 0;
2289 }
2290 if (uSelES & (X86_SEL_RPL | X86_SEL_LDT))
2291 {
2292 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_ES;
2293 pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2294 uSelES = 0;
2295 }
2296 if (uSelFS & (X86_SEL_RPL | X86_SEL_LDT))
2297 {
2298 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_FS;
2299 pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2300 uSelFS = 0;
2301 }
2302 if (uSelGS & (X86_SEL_RPL | X86_SEL_LDT))
2303 {
2304 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_GS;
2305 pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2306 uSelGS = 0;
2307 }
2308#endif
2309
2310 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2311 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2312 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2313 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2314 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2315 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2316 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2317 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2318 Assert(uSelCS);
2319 Assert(uSelTR);
2320
2321 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2322#if 0
2323 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2324 Assert(uSelSS != 0);
2325#endif
2326
2327 /* Write these host selector fields into the host-state area in the VMCS. */
2328 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2329 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2330#if HC_ARCH_BITS == 64
2331 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2332 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2333 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2334 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2335#endif
2336 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2337
2338 /*
2339 * Host GDTR and IDTR.
2340 */
2341 RTGDTR Gdtr;
2342 RT_ZERO(Gdtr);
2343#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2344 if (HMVMX_IS_64BIT_HOST_MODE())
2345 {
2346 X86XDTR64 Gdtr64;
2347 X86XDTR64 Idtr64;
2348 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2349 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2350 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2351
2352 Gdtr.cbGdt = Gdtr64.cb;
2353 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2354 }
2355 else
2356#endif
2357 {
2358 RTIDTR Idtr;
2359 ASMGetGDTR(&Gdtr);
2360 ASMGetIDTR(&Idtr);
2361 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2362 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2363
2364#if HC_ARCH_BITS == 64
2365 /*
2366 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2367 * maximum limit (0xffff) on every VM-exit.
2368 */
2369 if (Gdtr.cbGdt != 0xffff)
2370 {
2371 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2372 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2373 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2374 }
2375
2376 /*
2377 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2378 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2379 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2380 */
2381 if (Idtr.cbIdt < 0x0fff)
2382 {
2383 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2384 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2385 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2386 }
2387#endif
2388 }
2389
2390 /*
2391 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2392 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2393 */
2394 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2395 {
2396 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2397 return VERR_VMX_INVALID_HOST_STATE;
2398 }
2399
2400 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2401#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2402 if (HMVMX_IS_64BIT_HOST_MODE())
2403 {
2404 /* We need the 64-bit TR base for hybrid darwin. */
2405 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2406 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2407 }
2408 else
2409#endif
2410 {
2411 uintptr_t uTRBase;
2412#if HC_ARCH_BITS == 64
2413 uTRBase = X86DESC64_BASE(pDesc);
2414
2415 /*
2416 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2417 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2418 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2419 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2420 *
2421 * [1] See Intel spec. 3.5 "System Descriptor Types".
2422 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2423 */
2424 Assert(pDesc->System.u4Type == 11);
2425 if ( pDesc->System.u16LimitLow != 0x67
2426 || pDesc->System.u4LimitHigh)
2427 {
2428 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2429 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2430
2431 /* Store the GDTR here as we need it while restoring TR. */
2432 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2433 }
2434#else
2435 uTRBase = X86DESC_BASE(pDesc);
2436#endif
2437 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2438 }
2439 AssertRCReturn(rc, rc);
2440
2441 /*
2442 * Host FS base and GS base.
2443 */
2444#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2445 if (HMVMX_IS_64BIT_HOST_MODE())
2446 {
2447 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2448 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2449 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2450 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2451
2452# if HC_ARCH_BITS == 64
2453 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2454 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2455 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2456 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2457 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2458# endif
2459 }
2460#endif
2461 return rc;
2462}
2463
2464
2465/**
2466 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2467 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2468 * the host after every successful VM exit.
2469 *
2470 * @returns VBox status code.
2471 * @param pVM Pointer to the VM.
2472 * @param pVCpu Pointer to the VMCPU.
2473 */
2474DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2475{
2476 AssertPtr(pVCpu);
2477 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2478
2479 int rc = VINF_SUCCESS;
2480#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2481 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2482 uint32_t cHostMsrs = 0;
2483 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2484
2485 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2486 {
2487 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2488
2489# if HC_ARCH_BITS == 64
2490 /* Paranoia. 64-bit code requires these bits to be set always. */
2491 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2492
2493 /*
2494 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2495 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2496 * some reason (e.g. allow transparent reads) we would activate the code below.
2497 */
2498# if 0
2499 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2500 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2501 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2502 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2503 if (CPUMIsGuestInLongMode(pVCpu))
2504 {
2505 uint64_t u64GuestEfer;
2506 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2507 AssertRC(rc);
2508
2509 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2510 {
2511 pHostMsr->u32Msr = MSR_K6_EFER;
2512 pHostMsr->u32Reserved = 0;
2513 pHostMsr->u64Value = u64HostEfer;
2514 pHostMsr++; cHostMsrs++;
2515 }
2516 }
2517# endif
2518# else /* HC_ARCH_BITS != 64 */
2519 pHostMsr->u32Msr = MSR_K6_EFER;
2520 pHostMsr->u32Reserved = 0;
2521# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2522 if (CPUMIsGuestInLongMode(pVCpu))
2523 {
2524 /* Must match the EFER value in our 64 bits switcher. */
2525 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2526 }
2527 else
2528# endif
2529 pHostMsr->u64Value = u64HostEfer;
2530 pHostMsr++; cHostMsrs++;
2531# endif /* HC_ARCH_BITS == 64 */
2532 }
2533
2534# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2535 if (HMVMX_IS_64BIT_HOST_MODE())
2536 {
2537 pHostMsr->u32Msr = MSR_K6_STAR;
2538 pHostMsr->u32Reserved = 0;
2539 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2540 pHostMsr++; cHostMsrs++;
2541 pHostMsr->u32Msr = MSR_K8_LSTAR;
2542 pHostMsr->u32Reserved = 0;
2543 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2544 pHostMsr++; cHostMsrs++;
2545 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2546 pHostMsr->u32Reserved = 0;
2547 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2548 pHostMsr++; cHostMsrs++;
2549 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2550 pHostMsr->u32Reserved = 0;
2551 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2552 pHostMsr++; cHostMsrs++;
2553 }
2554# endif
2555
2556 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2557 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2558 {
2559 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2560 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2561 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2562 }
2563
2564 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2565#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2566
2567 /*
2568 * Host Sysenter MSRs.
2569 */
2570 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2571 AssertRCReturn(rc, rc);
2572#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2573 if (HMVMX_IS_64BIT_HOST_MODE())
2574 {
2575 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2576 AssertRCReturn(rc, rc);
2577 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2578 }
2579 else
2580 {
2581 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2582 AssertRCReturn(rc, rc);
2583 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2584 }
2585#elif HC_ARCH_BITS == 32
2586 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2587 AssertRCReturn(rc, rc);
2588 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2589#else
2590 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2591 AssertRCReturn(rc, rc);
2592 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2593#endif
2594 AssertRCReturn(rc, rc);
2595
2596 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2597 * hmR0VmxSetupExitCtls() !! */
2598 return rc;
2599}
2600
2601
2602/**
2603 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2604 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2605 * controls".
2606 *
2607 * @returns VBox status code.
2608 * @param pVCpu Pointer to the VMCPU.
2609 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2610 * out-of-sync. Make sure to update the required fields
2611 * before using them.
2612 *
2613 * @remarks No-long-jump zone!!!
2614 */
2615DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2616{
2617 int rc = VINF_SUCCESS;
2618 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2619 {
2620 PVM pVM = pVCpu->CTX_SUFF(pVM);
2621 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2622 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2623
2624 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2625 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2626
2627 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2628 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2629 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2630 else
2631 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2632
2633 /*
2634 * The following should -not- be set (since we're not in SMM mode):
2635 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2636 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2637 */
2638
2639 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2640 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2641 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2642
2643 if ((val & zap) != val)
2644 {
2645 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2646 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2647 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2648 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2649 }
2650
2651 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2652 AssertRCReturn(rc, rc);
2653
2654 /* Update VCPU with the currently set VM-exit controls. */
2655 pVCpu->hm.s.vmx.u32EntryCtls = val;
2656 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2657 }
2658 return rc;
2659}
2660
2661
2662/**
2663 * Sets up the VM-exit controls in the VMCS.
2664 *
2665 * @returns VBox status code.
2666 * @param pVM Pointer to the VM.
2667 * @param pVCpu Pointer to the VMCPU.
2668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2669 * out-of-sync. Make sure to update the required fields
2670 * before using them.
2671 *
2672 * @remarks requires EFER.
2673 */
2674DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2675{
2676 int rc = VINF_SUCCESS;
2677 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2678 {
2679 PVM pVM = pVCpu->CTX_SUFF(pVM);
2680 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2681 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2682
2683 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2684 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2685
2686 /*
2687 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2688 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2689 */
2690#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2691 if (HMVMX_IS_64BIT_HOST_MODE())
2692 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2693 else
2694 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2695#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2696 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2697 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2698 else
2699 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2700#endif
2701
2702 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2703 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2704
2705 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2706 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2707 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2708 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2709 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2710
2711 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2712 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2713
2714 if ((val & zap) != val)
2715 {
2716 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2717 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2718 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2719 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2720 }
2721
2722 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2723 AssertRCReturn(rc, rc);
2724
2725 /* Update VCPU with the currently set VM-exit controls. */
2726 pVCpu->hm.s.vmx.u32ExitCtls = val;
2727 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2728 }
2729 return rc;
2730}
2731
2732
2733/**
2734 * Loads the guest APIC and related state.
2735 *
2736 * @returns VBox status code.
2737 * @param pVM Pointer to the VM.
2738 * @param pVCpu Pointer to the VMCPU.
2739 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2740 * out-of-sync. Make sure to update the required fields
2741 * before using them.
2742 */
2743DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2744{
2745 int rc = VINF_SUCCESS;
2746 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2747 {
2748 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2749 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2750 {
2751 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2752
2753 bool fPendingIntr = false;
2754 uint8_t u8Tpr = 0;
2755 uint8_t u8PendingIntr = 0;
2756 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2757 AssertRCReturn(rc, rc);
2758
2759 /*
2760 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2761 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2762 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2763 * the interrupt when we VM-exit for other reasons.
2764 */
2765 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2766 uint32_t u32TprThreshold = 0;
2767 if (fPendingIntr)
2768 {
2769 /* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2770 const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2771 const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2772 if (u8PendingPriority <= u8TprPriority)
2773 u32TprThreshold = u8PendingPriority;
2774 else
2775 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2776 }
2777 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2778
2779 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2780 AssertRCReturn(rc, rc);
2781 }
2782
2783 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2784 }
2785 return rc;
2786}
2787
2788
2789/**
2790 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2791 *
2792 * @returns Guest's interruptibility-state.
2793 * @param pVCpu Pointer to the VMCPU.
2794 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2795 * out-of-sync. Make sure to update the required fields
2796 * before using them.
2797 *
2798 * @remarks No-long-jump zone!!!
2799 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2800 */
2801DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2802{
2803 /*
2804 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2805 * inhibit interrupts or clear any existing interrupt-inhibition.
2806 */
2807 uint32_t uIntrState = 0;
2808 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2809 {
2810 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2811 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
2812 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2813 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2814 {
2815 /*
2816 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2817 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2818 */
2819 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2820 }
2821 else if (pMixedCtx->eflags.Bits.u1IF)
2822 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2823 else
2824 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2825 }
2826 return uIntrState;
2827}
2828
2829
2830/**
2831 * Loads the guest's interruptibility-state into the guest-state area in the
2832 * VMCS.
2833 *
2834 * @returns VBox status code.
2835 * @param pVCpu Pointer to the VMCPU.
2836 * @param uIntrState The interruptibility-state to set.
2837 */
2838static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2839{
2840 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2841 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2842 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2843 AssertRCReturn(rc, rc);
2844 return rc;
2845}
2846
2847
2848/**
2849 * Loads the guest's RIP into the guest-state area in the VMCS.
2850 *
2851 * @returns VBox status code.
2852 * @param pVCpu Pointer to the VMCPU.
2853 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2854 * out-of-sync. Make sure to update the required fields
2855 * before using them.
2856 *
2857 * @remarks No-long-jump zone!!!
2858 */
2859static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2860{
2861 int rc = VINF_SUCCESS;
2862 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2863 {
2864 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2865 AssertRCReturn(rc, rc);
2866 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2867 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#x\n", pMixedCtx->rip, pVCpu->hm.s.fContextUseFlags));
2868 }
2869 return rc;
2870}
2871
2872
2873/**
2874 * Loads the guest's RSP into the guest-state area in the VMCS.
2875 *
2876 * @returns VBox status code.
2877 * @param pVCpu Pointer to the VMCPU.
2878 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2879 * out-of-sync. Make sure to update the required fields
2880 * before using them.
2881 *
2882 * @remarks No-long-jump zone!!!
2883 */
2884static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2885{
2886 int rc = VINF_SUCCESS;
2887 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2888 {
2889 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2890 AssertRCReturn(rc, rc);
2891 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2892 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2893 }
2894 return rc;
2895}
2896
2897
2898/**
2899 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
2900 *
2901 * @returns VBox status code.
2902 * @param pVCpu Pointer to the VMCPU.
2903 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2904 * out-of-sync. Make sure to update the required fields
2905 * before using them.
2906 *
2907 * @remarks No-long-jump zone!!!
2908 */
2909static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2910{
2911 int rc = VINF_SUCCESS;
2912 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2913 {
2914 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2915 Let us assert it as such and use 32-bit VMWRITE. */
2916 Assert(!(pMixedCtx->rflags.u64 >> 32));
2917 X86EFLAGS Eflags = pMixedCtx->eflags;
2918 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2919 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2920
2921 /*
2922 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2923 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2924 */
2925 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2926 {
2927 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2928 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2929 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2930 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2931 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2932 }
2933
2934 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2935 AssertRCReturn(rc, rc);
2936
2937 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2938 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2939 }
2940 return rc;
2941}
2942
2943
2944/**
2945 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2946 *
2947 * @returns VBox status code.
2948 * @param pVCpu Pointer to the VMCPU.
2949 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2950 * out-of-sync. Make sure to update the required fields
2951 * before using them.
2952 *
2953 * @remarks No-long-jump zone!!!
2954 */
2955DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2956{
2957 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2958 AssertRCReturn(rc, rc);
2959 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2960 AssertRCReturn(rc, rc);
2961 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2962 AssertRCReturn(rc, rc);
2963 return rc;
2964}
2965
2966
2967/**
2968 * Loads the guest CR0 control register into the guest-state area in the VMCS.
2969 * CR0 is partially shared with the host and we have to consider the FPU bits.
2970 *
2971 * @returns VBox status code.
2972 * @param pVM Pointer to the VM.
2973 * @param pVCpu Pointer to the VMCPU.
2974 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2975 * out-of-sync. Make sure to update the required fields
2976 * before using them.
2977 *
2978 * @remarks No-long-jump zone!!!
2979 */
2980static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2981{
2982 /*
2983 * Guest CR0.
2984 * Guest FPU.
2985 */
2986 int rc = VINF_SUCCESS;
2987 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2988 {
2989 Assert(!(pMixedCtx->cr0 >> 32));
2990 uint32_t u32GuestCR0 = pMixedCtx->cr0;
2991 PVM pVM = pVCpu->CTX_SUFF(pVM);
2992
2993 /* The guest's view (read access) of its CR0 is unblemished. */
2994 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2995 AssertRCReturn(rc, rc);
2996 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2997
2998 /* Setup VT-x's view of the guest CR0. */
2999 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3000 if (pVM->hm.s.fNestedPaging)
3001 {
3002 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3003 {
3004 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3005 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3006 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3007 }
3008 else
3009 {
3010 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3011 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3012 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3013 }
3014
3015 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3016 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3017 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3018
3019 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3020 AssertRCReturn(rc, rc);
3021 }
3022 else
3023 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3024
3025 /*
3026 * Guest FPU bits.
3027 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3028 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3029 */
3030 u32GuestCR0 |= X86_CR0_NE;
3031 bool fInterceptNM = false;
3032 if (CPUMIsGuestFPUStateActive(pVCpu))
3033 {
3034 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3035 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3036 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3037 }
3038 else
3039 {
3040 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3041 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3042 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3043 }
3044
3045 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3046 bool fInterceptMF = false;
3047 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3048 fInterceptMF = true;
3049
3050 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3051 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3052 {
3053 Assert(PDMVmmDevHeapIsEnabled(pVM));
3054 Assert(pVM->hm.s.vmx.pRealModeTSS);
3055 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3056 fInterceptNM = true;
3057 fInterceptMF = true;
3058 }
3059 else
3060 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3061
3062 if (fInterceptNM)
3063 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3064 else
3065 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3066
3067 if (fInterceptMF)
3068 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3069 else
3070 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3071
3072 /* Additional intercepts for debugging, define these yourself explicitly. */
3073#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3074 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3075 | RT_BIT(X86_XCPT_BP)
3076 | RT_BIT(X86_XCPT_DB)
3077 | RT_BIT(X86_XCPT_DE)
3078 | RT_BIT(X86_XCPT_NM)
3079 | RT_BIT(X86_XCPT_UD)
3080 | RT_BIT(X86_XCPT_NP)
3081 | RT_BIT(X86_XCPT_SS)
3082 | RT_BIT(X86_XCPT_GP)
3083 | RT_BIT(X86_XCPT_PF)
3084 | RT_BIT(X86_XCPT_MF)
3085 ;
3086#elif defined(HMVMX_ALWAYS_TRAP_PF)
3087 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3088#endif
3089
3090 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3091
3092 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3093 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3094 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3095 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3096 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3097 else
3098 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3099
3100 u32GuestCR0 |= uSetCR0;
3101 u32GuestCR0 &= uZapCR0;
3102 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3103
3104 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3105 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3106 AssertRCReturn(rc, rc);
3107 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3108 AssertRCReturn(rc, rc);
3109 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3110
3111 /*
3112 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3113 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3114 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3115 */
3116 uint32_t u32CR0Mask = 0;
3117 u32CR0Mask = X86_CR0_PE
3118 | X86_CR0_NE
3119 | X86_CR0_WP
3120 | X86_CR0_PG
3121 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3122 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3123 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3124
3125 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3126 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3127 * and @bugref{6944}. */
3128#if 0
3129 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3130 u32CR0Mask &= ~X86_CR0_PE;
3131#endif
3132 if (pVM->hm.s.fNestedPaging)
3133 u32CR0Mask &= ~X86_CR0_WP;
3134
3135 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3136 if (fInterceptNM)
3137 {
3138 u32CR0Mask |= X86_CR0_TS
3139 | X86_CR0_MP;
3140 }
3141
3142 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3143 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3144 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3145 AssertRCReturn(rc, rc);
3146 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3147
3148 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3149 }
3150 return rc;
3151}
3152
3153
3154/**
3155 * Loads the guest control registers (CR3, CR4) into the guest-state area
3156 * in the VMCS.
3157 *
3158 * @returns VBox status code.
3159 * @param pVM Pointer to the VM.
3160 * @param pVCpu Pointer to the VMCPU.
3161 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3162 * out-of-sync. Make sure to update the required fields
3163 * before using them.
3164 *
3165 * @remarks No-long-jump zone!!!
3166 */
3167static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3168{
3169 int rc = VINF_SUCCESS;
3170 PVM pVM = pVCpu->CTX_SUFF(pVM);
3171
3172 /*
3173 * Guest CR2.
3174 * It's always loaded in the assembler code. Nothing to do here.
3175 */
3176
3177 /*
3178 * Guest CR3.
3179 */
3180 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3181 {
3182 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3183 if (pVM->hm.s.fNestedPaging)
3184 {
3185 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3186
3187 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3188 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3189 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3190 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3191
3192 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3193 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3194 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3195
3196 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3197 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3198 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3199 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3200
3201 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3202 AssertRCReturn(rc, rc);
3203 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3204
3205 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3206 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3207 {
3208 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3209 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3210 {
3211 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3212 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3213 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3214 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3215 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3216 }
3217
3218 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3219 have Unrestricted Execution to handle the guest when it's not using paging. */
3220 GCPhysGuestCR3 = pMixedCtx->cr3;
3221 }
3222 else
3223 {
3224 /*
3225 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3226 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3227 * EPT takes care of translating it to host-physical addresses.
3228 */
3229 RTGCPHYS GCPhys;
3230 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3231 Assert(PDMVmmDevHeapIsEnabled(pVM));
3232
3233 /* We obtain it here every time as the guest could have relocated this PCI region. */
3234 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3235 AssertRCReturn(rc, rc);
3236
3237 GCPhysGuestCR3 = GCPhys;
3238 }
3239
3240 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3241 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3242 }
3243 else
3244 {
3245 /* Non-nested paging case, just use the hypervisor's CR3. */
3246 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3247
3248 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3249 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3250 }
3251 AssertRCReturn(rc, rc);
3252
3253 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3254 }
3255
3256 /*
3257 * Guest CR4.
3258 */
3259 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3260 {
3261 Assert(!(pMixedCtx->cr4 >> 32));
3262 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3263
3264 /* The guest's view of its CR4 is unblemished. */
3265 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3266 AssertRCReturn(rc, rc);
3267 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3268
3269 /* Setup VT-x's view of the guest CR4. */
3270 /*
3271 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3272 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3273 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3274 */
3275 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3276 {
3277 Assert(pVM->hm.s.vmx.pRealModeTSS);
3278 Assert(PDMVmmDevHeapIsEnabled(pVM));
3279 u32GuestCR4 &= ~X86_CR4_VME;
3280 }
3281
3282 if (pVM->hm.s.fNestedPaging)
3283 {
3284 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3285 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3286 {
3287 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3288 u32GuestCR4 |= X86_CR4_PSE;
3289 /* Our identity mapping is a 32 bits page directory. */
3290 u32GuestCR4 &= ~X86_CR4_PAE;
3291 }
3292 /* else use guest CR4.*/
3293 }
3294 else
3295 {
3296 /*
3297 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3298 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3299 */
3300 switch (pVCpu->hm.s.enmShadowMode)
3301 {
3302 case PGMMODE_REAL: /* Real-mode. */
3303 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3304 case PGMMODE_32_BIT: /* 32-bit paging. */
3305 {
3306 u32GuestCR4 &= ~X86_CR4_PAE;
3307 break;
3308 }
3309
3310 case PGMMODE_PAE: /* PAE paging. */
3311 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3312 {
3313 u32GuestCR4 |= X86_CR4_PAE;
3314 break;
3315 }
3316
3317 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3318 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3319#ifdef VBOX_ENABLE_64_BITS_GUESTS
3320 break;
3321#endif
3322 default:
3323 AssertFailed();
3324 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3325 }
3326 }
3327
3328 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3329 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3330 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3331 u32GuestCR4 |= uSetCR4;
3332 u32GuestCR4 &= uZapCR4;
3333
3334 /* Write VT-x's view of the guest CR4 into the VMCS. */
3335 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3336 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3337 AssertRCReturn(rc, rc);
3338
3339 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3340 uint32_t u32CR4Mask = 0;
3341 u32CR4Mask = X86_CR4_VME
3342 | X86_CR4_PAE
3343 | X86_CR4_PGE
3344 | X86_CR4_PSE
3345 | X86_CR4_VMXE;
3346 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3347 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3348 AssertRCReturn(rc, rc);
3349
3350 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3351 }
3352 return rc;
3353}
3354
3355
3356/**
3357 * Loads the guest debug registers into the guest-state area in the VMCS.
3358 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3359 *
3360 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3361 *
3362 * @returns VBox status code.
3363 * @param pVCpu Pointer to the VMCPU.
3364 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3365 * out-of-sync. Make sure to update the required fields
3366 * before using them.
3367 *
3368 * @remarks No-long-jump zone!!!
3369 */
3370static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3371{
3372 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3373 return VINF_SUCCESS;
3374
3375#ifdef VBOX_STRICT
3376 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3377 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3378 {
3379 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3380 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3381 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3382 }
3383#endif
3384
3385 int rc;
3386 PVM pVM = pVCpu->CTX_SUFF(pVM);
3387 bool fInterceptDB = false;
3388 bool fInterceptMovDRx = false;
3389 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3390 {
3391 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3392 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3393 {
3394 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3395 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3396 AssertRCReturn(rc, rc);
3397 Assert(fInterceptDB == false);
3398 }
3399 else
3400 {
3401 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3402 pVCpu->hm.s.fClearTrapFlag = true;
3403 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RFLAGS;
3404 fInterceptDB = true;
3405 }
3406 }
3407
3408 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3409 {
3410 /*
3411 * Use the combined guest and host DRx values found in the hypervisor
3412 * register set because the debugger has breakpoints active or someone
3413 * is single stepping on the host side without a monitor trap flag.
3414 *
3415 * Note! DBGF expects a clean DR6 state before executing guest code.
3416 */
3417#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3418 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3419 && !CPUMIsHyperDebugStateActivePending(pVCpu))
3420 {
3421 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3422 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
3423 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
3424 }
3425 else
3426#endif
3427 if (!CPUMIsHyperDebugStateActive(pVCpu))
3428 {
3429 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3430 Assert(CPUMIsHyperDebugStateActive(pVCpu));
3431 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3432 }
3433
3434 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3435 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3436 AssertRCReturn(rc, rc);
3437
3438 fInterceptDB = true;
3439 fInterceptMovDRx = true;
3440 }
3441 else
3442 {
3443 /*
3444 * If the guest has enabled debug registers, we need to load them prior to
3445 * executing guest code so they'll trigger at the right time.
3446 */
3447 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3448 {
3449#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3450 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3451 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3452 {
3453 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3454 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
3455 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
3456 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3457 }
3458 else
3459#endif
3460 if (CPUMIsGuestDebugStateActive(pVCpu))
3461 {
3462 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3463 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3464 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3465 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3466 }
3467 }
3468 /*
3469 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3470 * must intercept #DB in order to maintain a correct DR6 guest value.
3471 */
3472#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3473 else if ( ( CPUMIsGuestInLongModeEx(pMixedCtx)
3474 && !CPUMIsGuestDebugStateActivePending(pVCpu))
3475 || !CPUMIsGuestDebugStateActive(pVCpu))
3476#else
3477 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3478#endif
3479 {
3480 fInterceptMovDRx = true;
3481 fInterceptDB = true;
3482 }
3483
3484 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3485 AssertRCReturn(rc, rc);
3486 }
3487
3488 /*
3489 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3490 */
3491 if (fInterceptDB)
3492 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3493 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3494 {
3495#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3496 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3497#endif
3498 }
3499 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3500 AssertRCReturn(rc, rc);
3501
3502 /*
3503 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3504 */
3505 if (fInterceptMovDRx)
3506 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3507 else
3508 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3509 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3510 AssertRCReturn(rc, rc);
3511
3512 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3513 return VINF_SUCCESS;
3514}
3515
3516
3517#ifdef VBOX_STRICT
3518/**
3519 * Strict function to validate segment registers.
3520 *
3521 * @remarks ASSUMES CR0 is up to date.
3522 */
3523static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3524{
3525 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3526 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3527 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3528 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3529 && ( !CPUMIsGuestInRealModeEx(pCtx)
3530 && !CPUMIsGuestInV86ModeEx(pCtx)))
3531 {
3532 /* Protected mode checks */
3533 /* CS */
3534 Assert(pCtx->cs.Attr.n.u1Present);
3535 Assert(!(pCtx->cs.Attr.u & 0xf00));
3536 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3537 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3538 || !(pCtx->cs.Attr.n.u1Granularity));
3539 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3540 || (pCtx->cs.Attr.n.u1Granularity));
3541 /* CS cannot be loaded with NULL in protected mode. */
3542 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3543 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3544 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3545 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3546 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3547 else
3548 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3549 /* SS */
3550 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3551 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3552 if ( !(pCtx->cr0 & X86_CR0_PE)
3553 || pCtx->cs.Attr.n.u4Type == 3)
3554 {
3555 Assert(!pCtx->ss.Attr.n.u2Dpl);
3556 }
3557 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3558 {
3559 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3560 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3561 Assert(pCtx->ss.Attr.n.u1Present);
3562 Assert(!(pCtx->ss.Attr.u & 0xf00));
3563 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3564 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3565 || !(pCtx->ss.Attr.n.u1Granularity));
3566 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3567 || (pCtx->ss.Attr.n.u1Granularity));
3568 }
3569 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3570 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3571 {
3572 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3573 Assert(pCtx->ds.Attr.n.u1Present);
3574 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3575 Assert(!(pCtx->ds.Attr.u & 0xf00));
3576 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3577 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3578 || !(pCtx->ds.Attr.n.u1Granularity));
3579 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3580 || (pCtx->ds.Attr.n.u1Granularity));
3581 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3582 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3583 }
3584 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3585 {
3586 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3587 Assert(pCtx->es.Attr.n.u1Present);
3588 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3589 Assert(!(pCtx->es.Attr.u & 0xf00));
3590 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3591 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3592 || !(pCtx->es.Attr.n.u1Granularity));
3593 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3594 || (pCtx->es.Attr.n.u1Granularity));
3595 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3596 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3597 }
3598 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3599 {
3600 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3601 Assert(pCtx->fs.Attr.n.u1Present);
3602 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3603 Assert(!(pCtx->fs.Attr.u & 0xf00));
3604 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3605 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3606 || !(pCtx->fs.Attr.n.u1Granularity));
3607 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3608 || (pCtx->fs.Attr.n.u1Granularity));
3609 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3610 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3611 }
3612 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3613 {
3614 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3615 Assert(pCtx->gs.Attr.n.u1Present);
3616 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3617 Assert(!(pCtx->gs.Attr.u & 0xf00));
3618 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3619 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3620 || !(pCtx->gs.Attr.n.u1Granularity));
3621 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3622 || (pCtx->gs.Attr.n.u1Granularity));
3623 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3624 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3625 }
3626 /* 64-bit capable CPUs. */
3627# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3628 Assert(!(pCtx->cs.u64Base >> 32));
3629 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3630 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3631 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3632# endif
3633 }
3634 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3635 || ( CPUMIsGuestInRealModeEx(pCtx)
3636 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3637 {
3638 /* Real and v86 mode checks. */
3639 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3640 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3641 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3642 {
3643 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3644 }
3645 else
3646 {
3647 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3648 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3649 }
3650
3651 /* CS */
3652 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3653 Assert(pCtx->cs.u32Limit == 0xffff);
3654 Assert(u32CSAttr == 0xf3);
3655 /* SS */
3656 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3657 Assert(pCtx->ss.u32Limit == 0xffff);
3658 Assert(u32SSAttr == 0xf3);
3659 /* DS */
3660 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3661 Assert(pCtx->ds.u32Limit == 0xffff);
3662 Assert(u32DSAttr == 0xf3);
3663 /* ES */
3664 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3665 Assert(pCtx->es.u32Limit == 0xffff);
3666 Assert(u32ESAttr == 0xf3);
3667 /* FS */
3668 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3669 Assert(pCtx->fs.u32Limit == 0xffff);
3670 Assert(u32FSAttr == 0xf3);
3671 /* GS */
3672 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3673 Assert(pCtx->gs.u32Limit == 0xffff);
3674 Assert(u32GSAttr == 0xf3);
3675 /* 64-bit capable CPUs. */
3676# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3677 Assert(!(pCtx->cs.u64Base >> 32));
3678 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3679 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3680 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3681# endif
3682 }
3683}
3684#endif /* VBOX_STRICT */
3685
3686
3687/**
3688 * Writes a guest segment register into the guest-state area in the VMCS.
3689 *
3690 * @returns VBox status code.
3691 * @param pVCpu Pointer to the VMCPU.
3692 * @param idxSel Index of the selector in the VMCS.
3693 * @param idxLimit Index of the segment limit in the VMCS.
3694 * @param idxBase Index of the segment base in the VMCS.
3695 * @param idxAccess Index of the access rights of the segment in the VMCS.
3696 * @param pSelReg Pointer to the segment selector.
3697 * @param pCtx Pointer to the guest-CPU context.
3698 *
3699 * @remarks No-long-jump zone!!!
3700 */
3701static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3702 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3703{
3704 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3705 AssertRCReturn(rc, rc);
3706 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3707 AssertRCReturn(rc, rc);
3708 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3709 AssertRCReturn(rc, rc);
3710
3711 uint32_t u32Access = pSelReg->Attr.u;
3712 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3713 {
3714 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3715 u32Access = 0xf3;
3716 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3717 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3718 }
3719 else
3720 {
3721 /*
3722 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3723 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3724 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3725 * loaded in protected-mode have their attribute as 0.
3726 */
3727 if (!u32Access)
3728 u32Access = X86DESCATTR_UNUSABLE;
3729 }
3730
3731 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3732 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3733 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3734
3735 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3736 AssertRCReturn(rc, rc);
3737 return rc;
3738}
3739
3740
3741/**
3742 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3743 * into the guest-state area in the VMCS.
3744 *
3745 * @returns VBox status code.
3746 * @param pVM Pointer to the VM.
3747 * @param pVCPU Pointer to the VMCPU.
3748 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3749 * out-of-sync. Make sure to update the required fields
3750 * before using them.
3751 *
3752 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
3753 * @remarks No-long-jump zone!!!
3754 */
3755static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3756{
3757 int rc = VERR_INTERNAL_ERROR_5;
3758 PVM pVM = pVCpu->CTX_SUFF(pVM);
3759
3760 /*
3761 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
3762 */
3763 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3764 {
3765 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3766 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3767 {
3768 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3769 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3770 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3771 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3772 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3773 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3774 }
3775
3776#ifdef VBOX_WITH_REM
3777 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3778 {
3779 Assert(pVM->hm.s.vmx.pRealModeTSS);
3780 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3781 if ( pVCpu->hm.s.vmx.fWasInRealMode
3782 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3783 {
3784 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3785 in real-mode (e.g. OpenBSD 4.0) */
3786 REMFlushTBs(pVM);
3787 Log4(("Load: Switch to protected mode detected!\n"));
3788 pVCpu->hm.s.vmx.fWasInRealMode = false;
3789 }
3790 }
3791#endif
3792 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3793 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3794 AssertRCReturn(rc, rc);
3795 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3796 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3797 AssertRCReturn(rc, rc);
3798 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3799 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3800 AssertRCReturn(rc, rc);
3801 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3802 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3803 AssertRCReturn(rc, rc);
3804 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3805 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3806 AssertRCReturn(rc, rc);
3807 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3808 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3809 AssertRCReturn(rc, rc);
3810
3811 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3812 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3813#ifdef VBOX_STRICT
3814 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3815#endif
3816 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3817 }
3818
3819 /*
3820 * Guest TR.
3821 */
3822 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3823 {
3824 /*
3825 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3826 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3827 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3828 */
3829 uint16_t u16Sel = 0;
3830 uint32_t u32Limit = 0;
3831 uint64_t u64Base = 0;
3832 uint32_t u32AccessRights = 0;
3833
3834 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3835 {
3836 u16Sel = pMixedCtx->tr.Sel;
3837 u32Limit = pMixedCtx->tr.u32Limit;
3838 u64Base = pMixedCtx->tr.u64Base;
3839 u32AccessRights = pMixedCtx->tr.Attr.u;
3840 }
3841 else
3842 {
3843 Assert(pVM->hm.s.vmx.pRealModeTSS);
3844 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3845
3846 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3847 RTGCPHYS GCPhys;
3848 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3849 AssertRCReturn(rc, rc);
3850
3851 X86DESCATTR DescAttr;
3852 DescAttr.u = 0;
3853 DescAttr.n.u1Present = 1;
3854 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3855
3856 u16Sel = 0;
3857 u32Limit = HM_VTX_TSS_SIZE;
3858 u64Base = GCPhys; /* in real-mode phys = virt. */
3859 u32AccessRights = DescAttr.u;
3860 }
3861
3862 /* Validate. */
3863 Assert(!(u16Sel & RT_BIT(2)));
3864 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3865 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3866 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3867 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3868 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3869 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3870 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3871 Assert( (u32Limit & 0xfff) == 0xfff
3872 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3873 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3874 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3875
3876 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3877 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3878 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3879 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3880
3881 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3882 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3883 }
3884
3885 /*
3886 * Guest GDTR.
3887 */
3888 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3889 {
3890 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3891 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3892
3893 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3894 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3895 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3896 }
3897
3898 /*
3899 * Guest LDTR.
3900 */
3901 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3902 {
3903 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3904 uint32_t u32Access = 0;
3905 if (!pMixedCtx->ldtr.Attr.u)
3906 u32Access = X86DESCATTR_UNUSABLE;
3907 else
3908 u32Access = pMixedCtx->ldtr.Attr.u;
3909
3910 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3911 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3912 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3913 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3914
3915 /* Validate. */
3916 if (!(u32Access & X86DESCATTR_UNUSABLE))
3917 {
3918 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3919 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3920 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3921 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3922 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3923 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3924 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3925 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3926 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3927 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3928 }
3929
3930 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3931 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3932 }
3933
3934 /*
3935 * Guest IDTR.
3936 */
3937 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3938 {
3939 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3940 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3941
3942 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3943 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3944 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3945 }
3946
3947 return VINF_SUCCESS;
3948}
3949
3950
3951/**
3952 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3953 * areas. These MSRs will automatically be loaded to the host CPU on every
3954 * successful VM entry and stored from the host CPU on every successful VM exit.
3955 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
3956 *
3957 * @returns VBox status code.
3958 * @param pVCpu Pointer to the VMCPU.
3959 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3960 * out-of-sync. Make sure to update the required fields
3961 * before using them.
3962 *
3963 * @remarks No-long-jump zone!!!
3964 */
3965static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3966{
3967 AssertPtr(pVCpu);
3968 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3969
3970 /*
3971 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3972 */
3973 int rc = VINF_SUCCESS;
3974 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3975 {
3976#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3977 PVM pVM = pVCpu->CTX_SUFF(pVM);
3978 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3979 uint32_t cGuestMsrs = 0;
3980
3981 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3982 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3983 * when the guest really is in 64-bit mode. */
3984 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3985 if (fSupportsLongMode)
3986 {
3987 pGuestMsr->u32Msr = MSR_K8_LSTAR;
3988 pGuestMsr->u32Reserved = 0;
3989 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3990 pGuestMsr++; cGuestMsrs++;
3991 pGuestMsr->u32Msr = MSR_K6_STAR;
3992 pGuestMsr->u32Reserved = 0;
3993 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3994 pGuestMsr++; cGuestMsrs++;
3995 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
3996 pGuestMsr->u32Reserved = 0;
3997 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3998 pGuestMsr++; cGuestMsrs++;
3999 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
4000 pGuestMsr->u32Reserved = 0;
4001 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
4002 pGuestMsr++; cGuestMsrs++;
4003 }
4004
4005 /*
4006 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
4007 * load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
4008 */
4009 if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
4010 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
4011 {
4012 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
4013 pGuestMsr->u32Reserved = 0;
4014 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
4015 AssertRCReturn(rc, rc);
4016 pGuestMsr++; cGuestMsrs++;
4017 }
4018
4019 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
4020 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
4021 {
4022 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
4023 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
4024 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4025 }
4026
4027 /* Update the VCPU's copy of the guest MSR count. */
4028 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
4029 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4030 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
4031#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
4032
4033 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
4034 }
4035
4036 /*
4037 * Guest Sysenter MSRs.
4038 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4039 * VM-exits on WRMSRs for these MSRs.
4040 */
4041 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4042 {
4043 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4044 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
4045 }
4046 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4047 {
4048 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4049 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
4050 }
4051 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4052 {
4053 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4054 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
4055 }
4056
4057 return rc;
4058}
4059
4060
4061/**
4062 * Loads the guest activity state into the guest-state area in the VMCS.
4063 *
4064 * @returns VBox status code.
4065 * @param pVCpu Pointer to the VMCPU.
4066 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4067 * out-of-sync. Make sure to update the required fields
4068 * before using them.
4069 *
4070 * @remarks No-long-jump zone!!!
4071 */
4072static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4073{
4074 /** @todo See if we can make use of other states, e.g.
4075 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4076 int rc = VINF_SUCCESS;
4077 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
4078 {
4079 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4080 AssertRCReturn(rc, rc);
4081 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
4082 }
4083 return rc;
4084}
4085
4086
4087/**
4088 * Sets up the appropriate function to run guest code.
4089 *
4090 * @returns VBox status code.
4091 * @param pVCpu Pointer to the VMCPU.
4092 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4093 * out-of-sync. Make sure to update the required fields
4094 * before using them.
4095 *
4096 * @remarks No-long-jump zone!!!
4097 */
4098static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4099{
4100 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4101 {
4102#ifndef VBOX_ENABLE_64_BITS_GUESTS
4103 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4104#endif
4105 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4106#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4107 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4108 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4109 {
4110 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4111 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS;
4112 }
4113#else
4114 /* 64-bit host or hybrid host. */
4115 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4116#endif
4117 }
4118 else
4119 {
4120 /* Guest is not in long mode, use the 32-bit handler. */
4121#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4122 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4123 {
4124 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4125 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS;
4126 }
4127#else
4128 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4129#endif
4130 }
4131 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4132 return VINF_SUCCESS;
4133}
4134
4135
4136/**
4137 * Wrapper for running the guest code in VT-x.
4138 *
4139 * @returns VBox strict status code.
4140 * @param pVM Pointer to the VM.
4141 * @param pVCpu Pointer to the VMCPU.
4142 * @param pCtx Pointer to the guest-CPU context.
4143 *
4144 * @remarks No-long-jump zone!!!
4145 */
4146DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4147{
4148 /*
4149 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4150 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4151 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4152 */
4153 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4154 /** @todo Add stats for resume vs launch. */
4155#ifdef VBOX_WITH_KERNEL_USING_XMM
4156 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4157#else
4158 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4159#endif
4160}
4161
4162
4163/**
4164 * Reports world-switch error and dumps some useful debug info.
4165 *
4166 * @param pVM Pointer to the VM.
4167 * @param pVCpu Pointer to the VMCPU.
4168 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4169 * @param pCtx Pointer to the guest-CPU context.
4170 * @param pVmxTransient Pointer to the VMX transient structure (only
4171 * exitReason updated).
4172 */
4173static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4174{
4175 Assert(pVM);
4176 Assert(pVCpu);
4177 Assert(pCtx);
4178 Assert(pVmxTransient);
4179 HMVMX_ASSERT_PREEMPT_SAFE();
4180
4181 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4182 switch (rcVMRun)
4183 {
4184 case VERR_VMX_INVALID_VMXON_PTR:
4185 AssertFailed();
4186 break;
4187 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4188 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4189 {
4190 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4191 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4192 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4193 AssertRC(rc);
4194
4195 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4196 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4197 Cannot do it here as we may have been long preempted. */
4198
4199#ifdef VBOX_STRICT
4200 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4201 pVmxTransient->uExitReason));
4202 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4203 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4204 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4205 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4206 else
4207 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4208 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4209 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4210
4211 /* VMX control bits. */
4212 uint32_t u32Val;
4213 uint64_t u64Val;
4214 HMVMXHCUINTREG uHCReg;
4215 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4216 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4217 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4218 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4219 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4220 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4221 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4222 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4223 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4224 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4225 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4226 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4227 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4228 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4229 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4230 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4231 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4232 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4233 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4234 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4235 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4236 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4237 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4238 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4239 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4240 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4241 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4242 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4243 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4244 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4245 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4246 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4247 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4248 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4249 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4250 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4251 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4252 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4253 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4254 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4255 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4256 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4257
4258 /* Guest bits. */
4259 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4260 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4261 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4262 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4263 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4264 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4265 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4266 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4267
4268 /* Host bits. */
4269 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4270 Log4(("Host CR0 %#RHr\n", uHCReg));
4271 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4272 Log4(("Host CR3 %#RHr\n", uHCReg));
4273 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4274 Log4(("Host CR4 %#RHr\n", uHCReg));
4275
4276 RTGDTR HostGdtr;
4277 PCX86DESCHC pDesc;
4278 ASMGetGDTR(&HostGdtr);
4279 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4280 Log4(("Host CS %#08x\n", u32Val));
4281 if (u32Val < HostGdtr.cbGdt)
4282 {
4283 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4284 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4285 }
4286
4287 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4288 Log4(("Host DS %#08x\n", u32Val));
4289 if (u32Val < HostGdtr.cbGdt)
4290 {
4291 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4292 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4293 }
4294
4295 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4296 Log4(("Host ES %#08x\n", u32Val));
4297 if (u32Val < HostGdtr.cbGdt)
4298 {
4299 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4300 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4301 }
4302
4303 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4304 Log4(("Host FS %#08x\n", u32Val));
4305 if (u32Val < HostGdtr.cbGdt)
4306 {
4307 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4308 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4309 }
4310
4311 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4312 Log4(("Host GS %#08x\n", u32Val));
4313 if (u32Val < HostGdtr.cbGdt)
4314 {
4315 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4316 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4317 }
4318
4319 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4320 Log4(("Host SS %#08x\n", u32Val));
4321 if (u32Val < HostGdtr.cbGdt)
4322 {
4323 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4324 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4325 }
4326
4327 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4328 Log4(("Host TR %#08x\n", u32Val));
4329 if (u32Val < HostGdtr.cbGdt)
4330 {
4331 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4332 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4333 }
4334
4335 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4336 Log4(("Host TR Base %#RHv\n", uHCReg));
4337 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4338 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4339 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4340 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4341 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4342 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4343 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4344 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4345 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4346 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4347 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4348 Log4(("Host RSP %#RHv\n", uHCReg));
4349 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4350 Log4(("Host RIP %#RHv\n", uHCReg));
4351# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4352 if (HMVMX_IS_64BIT_HOST_MODE())
4353 {
4354 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4355 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4356 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4357 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4358 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4359 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4360 }
4361# endif
4362#endif /* VBOX_STRICT */
4363 break;
4364 }
4365
4366 default:
4367 /* Impossible */
4368 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4369 break;
4370 }
4371 NOREF(pVM);
4372}
4373
4374
4375#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4376#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4377# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4378#endif
4379#ifdef VBOX_STRICT
4380static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4381{
4382 switch (idxField)
4383 {
4384 case VMX_VMCS_GUEST_RIP:
4385 case VMX_VMCS_GUEST_RSP:
4386 case VMX_VMCS_GUEST_SYSENTER_EIP:
4387 case VMX_VMCS_GUEST_SYSENTER_ESP:
4388 case VMX_VMCS_GUEST_GDTR_BASE:
4389 case VMX_VMCS_GUEST_IDTR_BASE:
4390 case VMX_VMCS_GUEST_CS_BASE:
4391 case VMX_VMCS_GUEST_DS_BASE:
4392 case VMX_VMCS_GUEST_ES_BASE:
4393 case VMX_VMCS_GUEST_FS_BASE:
4394 case VMX_VMCS_GUEST_GS_BASE:
4395 case VMX_VMCS_GUEST_SS_BASE:
4396 case VMX_VMCS_GUEST_LDTR_BASE:
4397 case VMX_VMCS_GUEST_TR_BASE:
4398 case VMX_VMCS_GUEST_CR3:
4399 return true;
4400 }
4401 return false;
4402}
4403
4404static bool hmR0VmxIsValidReadField(uint32_t idxField)
4405{
4406 switch (idxField)
4407 {
4408 /* Read-only fields. */
4409 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4410 return true;
4411 }
4412 /* Remaining readable fields should also be writable. */
4413 return hmR0VmxIsValidWriteField(idxField);
4414}
4415#endif /* VBOX_STRICT */
4416
4417
4418/**
4419 * Executes the specified handler in 64-bit mode.
4420 *
4421 * @returns VBox status code.
4422 * @param pVM Pointer to the VM.
4423 * @param pVCpu Pointer to the VMCPU.
4424 * @param pCtx Pointer to the guest CPU context.
4425 * @param enmOp The operation to perform.
4426 * @param cbParam Number of parameters.
4427 * @param paParam Array of 32-bit parameters.
4428 */
4429VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4430 uint32_t *paParam)
4431{
4432 int rc, rc2;
4433 PHMGLOBALCPUINFO pCpu;
4434 RTHCPHYS HCPhysCpuPage;
4435 RTCCUINTREG uOldEflags;
4436
4437 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4438 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4439 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4440 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4441
4442#ifdef VBOX_STRICT
4443 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4444 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4445
4446 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4447 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4448#endif
4449
4450 /* Disable interrupts. */
4451 uOldEflags = ASMIntDisableFlags();
4452
4453#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4454 RTCPUID idHostCpu = RTMpCpuId();
4455 CPUMR0SetLApic(pVCpu, idHostCpu);
4456#endif
4457
4458 pCpu = HMR0GetCurrentCpu();
4459 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4460
4461 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4462 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4463
4464 /* Leave VMX Root Mode. */
4465 VMXDisable();
4466
4467 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4468
4469 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4470 CPUMSetHyperEIP(pVCpu, enmOp);
4471 for (int i = (int)cbParam - 1; i >= 0; i--)
4472 CPUMPushHyper(pVCpu, paParam[i]);
4473
4474 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4475
4476 /* Call the switcher. */
4477 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4478 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4479
4480 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4481 /* Make sure the VMX instructions don't cause #UD faults. */
4482 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4483
4484 /* Re-enter VMX Root Mode */
4485 rc2 = VMXEnable(HCPhysCpuPage);
4486 if (RT_FAILURE(rc2))
4487 {
4488 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4489 ASMSetFlags(uOldEflags);
4490 return rc2;
4491 }
4492
4493 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4494 AssertRC(rc2);
4495 Assert(!(ASMGetFlags() & X86_EFL_IF));
4496 ASMSetFlags(uOldEflags);
4497 return rc;
4498}
4499
4500
4501/**
4502 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4503 * supporting 64-bit guests.
4504 *
4505 * @returns VBox status code.
4506 * @param fResume Whether to VMLAUNCH or VMRESUME.
4507 * @param pCtx Pointer to the guest-CPU context.
4508 * @param pCache Pointer to the VMCS cache.
4509 * @param pVM Pointer to the VM.
4510 * @param pVCpu Pointer to the VMCPU.
4511 */
4512DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4513{
4514 uint32_t aParam[6];
4515 PHMGLOBALCPUINFO pCpu = NULL;
4516 RTHCPHYS HCPhysCpuPage = 0;
4517 int rc = VERR_INTERNAL_ERROR_5;
4518
4519 pCpu = HMR0GetCurrentCpu();
4520 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4521
4522#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4523 pCache->uPos = 1;
4524 pCache->interPD = PGMGetInterPaeCR3(pVM);
4525 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4526#endif
4527
4528#ifdef VBOX_STRICT
4529 pCache->TestIn.HCPhysCpuPage = 0;
4530 pCache->TestIn.HCPhysVmcs = 0;
4531 pCache->TestIn.pCache = 0;
4532 pCache->TestOut.HCPhysVmcs = 0;
4533 pCache->TestOut.pCache = 0;
4534 pCache->TestOut.pCtx = 0;
4535 pCache->TestOut.eflags = 0;
4536#endif
4537
4538 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4539 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4540 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4541 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4542 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4543 aParam[5] = 0;
4544
4545#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4546 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4547 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4548#endif
4549 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4550
4551#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4552 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4553 Assert(pCtx->dr[4] == 10);
4554 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4555#endif
4556
4557#ifdef VBOX_STRICT
4558 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4559 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4560 pVCpu->hm.s.vmx.HCPhysVmcs));
4561 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4562 pCache->TestOut.HCPhysVmcs));
4563 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4564 pCache->TestOut.pCache));
4565 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4566 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4567 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4568 pCache->TestOut.pCtx));
4569 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4570#endif
4571 return rc;
4572}
4573
4574
4575/**
4576 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4577 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4578 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4579 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4580 *
4581 * @returns VBox status code.
4582 * @param pVM Pointer to the VM.
4583 * @param pVCpu Pointer to the VMCPU.
4584 */
4585static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4586{
4587#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4588{ \
4589 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4590 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4591 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4592 ++cReadFields; \
4593}
4594
4595 AssertPtr(pVM);
4596 AssertPtr(pVCpu);
4597 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4598 uint32_t cReadFields = 0;
4599
4600 /*
4601 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4602 * and serve to indicate exceptions to the rules.
4603 */
4604
4605 /* Guest-natural selector base fields. */
4606#if 0
4607 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4608 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4609 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4610#endif
4611 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4612 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4613 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4614 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4615 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4616 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4617 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4618 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4619 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4620 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4621 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4622 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4623#if 0
4624 /* Unused natural width guest-state fields. */
4625 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4626 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4627#endif
4628 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4629 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4630
4631 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4632#if 0
4633 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4634 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4635 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4636 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4637 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4638 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4639 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4640 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4641 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4642#endif
4643
4644 /* Natural width guest-state fields. */
4645 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4646#if 0
4647 /* Currently unused field. */
4648 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4649#endif
4650
4651 if (pVM->hm.s.fNestedPaging)
4652 {
4653 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4654 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4655 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4656 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4657 }
4658 else
4659 {
4660 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4661 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4662 }
4663
4664#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4665 return VINF_SUCCESS;
4666}
4667
4668
4669/**
4670 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4671 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4672 * darwin, running 64-bit guests).
4673 *
4674 * @returns VBox status code.
4675 * @param pVCpu Pointer to the VMCPU.
4676 * @param idxField The VMCS field encoding.
4677 * @param u64Val 16, 32 or 64 bits value.
4678 */
4679VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4680{
4681 int rc;
4682 switch (idxField)
4683 {
4684 /*
4685 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4686 */
4687 /* 64-bit Control fields. */
4688 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4689 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4690 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4691 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4692 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4693 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4694 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4695 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4696 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4697 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4698 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4699 case VMX_VMCS64_CTRL_EPTP_FULL:
4700 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4701 /* 64-bit Guest-state fields. */
4702 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4703 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4704 case VMX_VMCS64_GUEST_PAT_FULL:
4705 case VMX_VMCS64_GUEST_EFER_FULL:
4706 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4707 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4708 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4709 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4710 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4711 /* 64-bit Host-state fields. */
4712 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4713 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4714 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4715 {
4716 rc = VMXWriteVmcs32(idxField, u64Val);
4717 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4718 break;
4719 }
4720
4721 /*
4722 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4723 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4724 */
4725 /* Natural-width Guest-state fields. */
4726 case VMX_VMCS_GUEST_CR3:
4727 case VMX_VMCS_GUEST_ES_BASE:
4728 case VMX_VMCS_GUEST_CS_BASE:
4729 case VMX_VMCS_GUEST_SS_BASE:
4730 case VMX_VMCS_GUEST_DS_BASE:
4731 case VMX_VMCS_GUEST_FS_BASE:
4732 case VMX_VMCS_GUEST_GS_BASE:
4733 case VMX_VMCS_GUEST_LDTR_BASE:
4734 case VMX_VMCS_GUEST_TR_BASE:
4735 case VMX_VMCS_GUEST_GDTR_BASE:
4736 case VMX_VMCS_GUEST_IDTR_BASE:
4737 case VMX_VMCS_GUEST_RSP:
4738 case VMX_VMCS_GUEST_RIP:
4739 case VMX_VMCS_GUEST_SYSENTER_ESP:
4740 case VMX_VMCS_GUEST_SYSENTER_EIP:
4741 {
4742 if (!(u64Val >> 32))
4743 {
4744 /* If this field is 64-bit, VT-x will zero out the top bits. */
4745 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4746 }
4747 else
4748 {
4749 /* Assert that only the 32->64 switcher case should ever come here. */
4750 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4751 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4752 }
4753 break;
4754 }
4755
4756 default:
4757 {
4758 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4759 rc = VERR_INVALID_PARAMETER;
4760 break;
4761 }
4762 }
4763 AssertRCReturn(rc, rc);
4764 return rc;
4765}
4766
4767
4768/**
4769 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4770 * hosts (except darwin) for 64-bit guests.
4771 *
4772 * @param pVCpu Pointer to the VMCPU.
4773 * @param idxField The VMCS field encoding.
4774 * @param u64Val 16, 32 or 64 bits value.
4775 */
4776VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4777{
4778 AssertPtr(pVCpu);
4779 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4780
4781 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4782 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4783
4784 /* Make sure there are no duplicates. */
4785 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4786 {
4787 if (pCache->Write.aField[i] == idxField)
4788 {
4789 pCache->Write.aFieldVal[i] = u64Val;
4790 return VINF_SUCCESS;
4791 }
4792 }
4793
4794 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4795 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4796 pCache->Write.cValidEntries++;
4797 return VINF_SUCCESS;
4798}
4799
4800/* Enable later when the assembly code uses these as callbacks. */
4801#if 0
4802/*
4803 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4804 *
4805 * @param pVCpu Pointer to the VMCPU.
4806 * @param pCache Pointer to the VMCS cache.
4807 *
4808 * @remarks No-long-jump zone!!!
4809 */
4810VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4811{
4812 AssertPtr(pCache);
4813 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4814 {
4815 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4816 AssertRC(rc);
4817 }
4818 pCache->Write.cValidEntries = 0;
4819}
4820
4821
4822/**
4823 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
4824 *
4825 * @param pVCpu Pointer to the VMCPU.
4826 * @param pCache Pointer to the VMCS cache.
4827 *
4828 * @remarks No-long-jump zone!!!
4829 */
4830VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4831{
4832 AssertPtr(pCache);
4833 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4834 {
4835 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4836 AssertRC(rc);
4837 }
4838}
4839#endif
4840#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4841
4842
4843/**
4844 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4845 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4846 * timer.
4847 *
4848 * @returns VBox status code.
4849 * @param pVCpu Pointer to the VMCPU.
4850 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4851 * out-of-sync. Make sure to update the required fields
4852 * before using them.
4853 * @remarks No-long-jump zone!!!
4854 */
4855static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4856{
4857 int rc = VERR_INTERNAL_ERROR_5;
4858 bool fOffsettedTsc = false;
4859 PVM pVM = pVCpu->CTX_SUFF(pVM);
4860 if (pVM->hm.s.vmx.fUsePreemptTimer)
4861 {
4862 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4863
4864 /* Make sure the returned values have sane upper and lower boundaries. */
4865 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4866 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4867 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4868 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4869
4870 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4871 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4872 }
4873 else
4874 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4875
4876 if (fOffsettedTsc)
4877 {
4878 uint64_t u64CurTSC = ASMReadTSC();
4879 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4880 {
4881 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4882 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4883
4884 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4885 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4886 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4887 }
4888 else
4889 {
4890 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4891 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4892 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4893 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4894 }
4895 }
4896 else
4897 {
4898 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4899 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4900 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4901 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4902 }
4903}
4904
4905
4906/**
4907 * Determines if an exception is a contributory exception. Contributory
4908 * exceptions are ones which can cause double-faults. Page-fault is
4909 * intentionally not included here as it's a conditional contributory exception.
4910 *
4911 * @returns true if the exception is contributory, false otherwise.
4912 * @param uVector The exception vector.
4913 */
4914DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4915{
4916 switch (uVector)
4917 {
4918 case X86_XCPT_GP:
4919 case X86_XCPT_SS:
4920 case X86_XCPT_NP:
4921 case X86_XCPT_TS:
4922 case X86_XCPT_DE:
4923 return true;
4924 default:
4925 break;
4926 }
4927 return false;
4928}
4929
4930
4931/**
4932 * Sets an event as a pending event to be injected into the guest.
4933 *
4934 * @param pVCpu Pointer to the VMCPU.
4935 * @param u32IntrInfo The VM-entry interruption-information field.
4936 * @param cbInstr The VM-entry instruction length in bytes (for software
4937 * interrupts, exceptions and privileged software
4938 * exceptions).
4939 * @param u32ErrCode The VM-entry exception error code.
4940 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4941 * page-fault.
4942 *
4943 * @remarks Statistics counter assumes this is a guest event being injected or
4944 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4945 * always incremented.
4946 */
4947DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4948 RTGCUINTPTR GCPtrFaultAddress)
4949{
4950 Assert(!pVCpu->hm.s.Event.fPending);
4951 pVCpu->hm.s.Event.fPending = true;
4952 pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4953 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4954 pVCpu->hm.s.Event.cbInstr = cbInstr;
4955 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4956
4957 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4958}
4959
4960
4961/**
4962 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4963 *
4964 * @param pVCpu Pointer to the VMCPU.
4965 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4966 * out-of-sync. Make sure to update the required fields
4967 * before using them.
4968 */
4969DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4970{
4971 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
4972 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4973 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4974 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
4975}
4976
4977
4978/**
4979 * Handle a condition that occurred while delivering an event through the guest
4980 * IDT.
4981 *
4982 * @returns VBox status code (informational error codes included).
4983 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
4984 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4985 * continue execution of the guest which will delivery the #DF.
4986 * @retval VINF_EM_RESET if we detected a triple-fault condition.
4987 *
4988 * @param pVCpu Pointer to the VMCPU.
4989 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4990 * out-of-sync. Make sure to update the required fields
4991 * before using them.
4992 * @param pVmxTransient Pointer to the VMX transient structure.
4993 *
4994 * @remarks No-long-jump zone!!!
4995 */
4996static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4997{
4998 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4999 AssertRCReturn(rc, rc);
5000 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5001 {
5002 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
5003 AssertRCReturn(rc, rc);
5004
5005 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5006 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
5007 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5008
5009 typedef enum
5010 {
5011 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5012 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5013 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5014 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5015 } VMXREFLECTXCPT;
5016
5017 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5018 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5019 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
5020 {
5021 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5022 {
5023 enmReflect = VMXREFLECTXCPT_XCPT;
5024#ifdef VBOX_STRICT
5025 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5026 && uExitVector == X86_XCPT_PF)
5027 {
5028 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5029 }
5030#endif
5031 if ( uExitVector == X86_XCPT_PF
5032 && uIdtVector == X86_XCPT_PF)
5033 {
5034 pVmxTransient->fVectoringPF = true;
5035 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5036 }
5037 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5038 && hmR0VmxIsContributoryXcpt(uExitVector)
5039 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5040 || uIdtVector == X86_XCPT_PF))
5041 {
5042 enmReflect = VMXREFLECTXCPT_DF;
5043 }
5044 else if (uIdtVector == X86_XCPT_DF)
5045 enmReflect = VMXREFLECTXCPT_TF;
5046 }
5047 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5048 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5049 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5050 {
5051 /*
5052 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5053 * (whatever they are) as they reoccur when restarting the instruction.
5054 */
5055 enmReflect = VMXREFLECTXCPT_XCPT;
5056 }
5057 }
5058 else
5059 {
5060 /*
5061 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5062 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5063 * original exception to the guest after handling the VM-exit.
5064 */
5065 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5066 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5067 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5068 {
5069 enmReflect = VMXREFLECTXCPT_XCPT;
5070 }
5071 }
5072
5073 switch (enmReflect)
5074 {
5075 case VMXREFLECTXCPT_XCPT:
5076 {
5077 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5078 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5079 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5080
5081 uint32_t u32ErrCode = 0;
5082 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5083 {
5084 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5085 AssertRCReturn(rc, rc);
5086 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5087 }
5088
5089 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5090 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5091 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5092 rc = VINF_SUCCESS;
5093 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5094 pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
5095
5096 break;
5097 }
5098
5099 case VMXREFLECTXCPT_DF:
5100 {
5101 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5102 rc = VINF_HM_DOUBLE_FAULT;
5103 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5104 pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
5105
5106 break;
5107 }
5108
5109 case VMXREFLECTXCPT_TF:
5110 {
5111 rc = VINF_EM_RESET;
5112 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5113 uExitVector));
5114 break;
5115 }
5116
5117 default:
5118 Assert(rc == VINF_SUCCESS);
5119 break;
5120 }
5121 }
5122 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5123 return rc;
5124}
5125
5126
5127/**
5128 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5129 *
5130 * @returns VBox status code.
5131 * @param pVCpu Pointer to the VMCPU.
5132 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5133 * out-of-sync. Make sure to update the required fields
5134 * before using them.
5135 *
5136 * @remarks No-long-jump zone!!!
5137 */
5138static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5139{
5140 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5141 {
5142 uint32_t uVal = 0;
5143 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5144 AssertRCReturn(rc, rc);
5145
5146 uint32_t uShadow = 0;
5147 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5148 AssertRCReturn(rc, rc);
5149
5150 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5151 CPUMSetGuestCR0(pVCpu, uVal);
5152 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5153 }
5154 return VINF_SUCCESS;
5155}
5156
5157
5158/**
5159 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5160 *
5161 * @returns VBox status code.
5162 * @param pVCpu Pointer to the VMCPU.
5163 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5164 * out-of-sync. Make sure to update the required fields
5165 * before using them.
5166 *
5167 * @remarks No-long-jump zone!!!
5168 */
5169static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5170{
5171 int rc = VINF_SUCCESS;
5172 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5173 {
5174 uint32_t uVal = 0;
5175 uint32_t uShadow = 0;
5176 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5177 AssertRCReturn(rc, rc);
5178 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5179 AssertRCReturn(rc, rc);
5180
5181 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5182 CPUMSetGuestCR4(pVCpu, uVal);
5183 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5184 }
5185 return rc;
5186}
5187
5188
5189/**
5190 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5191 *
5192 * @returns VBox status code.
5193 * @param pVCpu Pointer to the VMCPU.
5194 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5195 * out-of-sync. Make sure to update the required fields
5196 * before using them.
5197 *
5198 * @remarks No-long-jump zone!!!
5199 */
5200static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5201{
5202 int rc = VINF_SUCCESS;
5203 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5204 {
5205 uint64_t u64Val = 0;
5206 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5207 AssertRCReturn(rc, rc);
5208
5209 pMixedCtx->rip = u64Val;
5210 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5211 }
5212 return rc;
5213}
5214
5215
5216/**
5217 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5218 *
5219 * @returns VBox status code.
5220 * @param pVCpu Pointer to the VMCPU.
5221 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5222 * out-of-sync. Make sure to update the required fields
5223 * before using them.
5224 *
5225 * @remarks No-long-jump zone!!!
5226 */
5227static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5228{
5229 int rc = VINF_SUCCESS;
5230 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5231 {
5232 uint64_t u64Val = 0;
5233 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5234 AssertRCReturn(rc, rc);
5235
5236 pMixedCtx->rsp = u64Val;
5237 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5238 }
5239 return rc;
5240}
5241
5242
5243/**
5244 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5245 *
5246 * @returns VBox status code.
5247 * @param pVCpu Pointer to the VMCPU.
5248 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5249 * out-of-sync. Make sure to update the required fields
5250 * before using them.
5251 *
5252 * @remarks No-long-jump zone!!!
5253 */
5254static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5255{
5256 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5257 {
5258 uint32_t uVal = 0;
5259 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5260 AssertRCReturn(rc, rc);
5261
5262 pMixedCtx->eflags.u32 = uVal;
5263 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5264 {
5265 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5266 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5267
5268 pMixedCtx->eflags.Bits.u1VM = 0;
5269 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5270 }
5271
5272 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5273 }
5274 return VINF_SUCCESS;
5275}
5276
5277
5278/**
5279 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5280 * guest-CPU context.
5281 */
5282DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5283{
5284 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5285 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5286 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5287 return rc;
5288}
5289
5290
5291/**
5292 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5293 * from the guest-state area in the VMCS.
5294 *
5295 * @param pVCpu Pointer to the VMCPU.
5296 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5297 * out-of-sync. Make sure to update the required fields
5298 * before using them.
5299 *
5300 * @remarks No-long-jump zone!!!
5301 */
5302static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5303{
5304 uint32_t uIntrState = 0;
5305 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5306 AssertRC(rc);
5307
5308 if (!uIntrState)
5309 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5310 else
5311 {
5312 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5313 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5314 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5315 AssertRC(rc);
5316 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5317 AssertRC(rc);
5318
5319 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5320 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5321 }
5322}
5323
5324
5325/**
5326 * Saves the guest's activity state.
5327 *
5328 * @returns VBox status code.
5329 * @param pVCpu Pointer to the VMCPU.
5330 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5331 * out-of-sync. Make sure to update the required fields
5332 * before using them.
5333 *
5334 * @remarks No-long-jump zone!!!
5335 */
5336static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5337{
5338 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5339 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5340 return VINF_SUCCESS;
5341}
5342
5343
5344/**
5345 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5346 * the current VMCS into the guest-CPU context.
5347 *
5348 * @returns VBox status code.
5349 * @param pVCpu Pointer to the VMCPU.
5350 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5351 * out-of-sync. Make sure to update the required fields
5352 * before using them.
5353 *
5354 * @remarks No-long-jump zone!!!
5355 */
5356static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5357{
5358 int rc = VINF_SUCCESS;
5359 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5360 {
5361 uint32_t u32Val = 0;
5362 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5363 pMixedCtx->SysEnter.cs = u32Val;
5364 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5365 }
5366
5367 uint64_t u64Val = 0;
5368 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5369 {
5370 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5371 pMixedCtx->SysEnter.eip = u64Val;
5372 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5373 }
5374 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5375 {
5376 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5377 pMixedCtx->SysEnter.esp = u64Val;
5378 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5379 }
5380 return rc;
5381}
5382
5383
5384/**
5385 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5386 * context.
5387 *
5388 * @returns VBox status code.
5389 * @param pVCpu Pointer to the VMCPU.
5390 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5391 * out-of-sync. Make sure to update the required fields
5392 * before using them.
5393 *
5394 * @remarks No-long-jump zone!!!
5395 */
5396static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5397{
5398 int rc = VINF_SUCCESS;
5399 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5400 {
5401 uint64_t u64Val = 0;
5402 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5403 pMixedCtx->fs.u64Base = u64Val;
5404 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5405 }
5406 return rc;
5407}
5408
5409
5410/**
5411 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5412 * context.
5413 *
5414 * @returns VBox status code.
5415 * @param pVCpu Pointer to the VMCPU.
5416 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5417 * out-of-sync. Make sure to update the required fields
5418 * before using them.
5419 *
5420 * @remarks No-long-jump zone!!!
5421 */
5422static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5423{
5424 int rc = VINF_SUCCESS;
5425 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5426 {
5427 uint64_t u64Val = 0;
5428 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5429 pMixedCtx->gs.u64Base = u64Val;
5430 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5431 }
5432 return rc;
5433}
5434
5435
5436/**
5437 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5438 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5439 * and TSC_AUX.
5440 *
5441 * @returns VBox status code.
5442 * @param pVCpu Pointer to the VMCPU.
5443 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5444 * out-of-sync. Make sure to update the required fields
5445 * before using them.
5446 *
5447 * @remarks No-long-jump zone!!!
5448 */
5449static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5450{
5451 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5452 return VINF_SUCCESS;
5453
5454#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5455 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5456 {
5457 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5458 pMsr += i;
5459 switch (pMsr->u32Msr)
5460 {
5461 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5462 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5463 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5464 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5465 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5466 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5467 default:
5468 {
5469 AssertFailed();
5470 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5471 }
5472 }
5473 }
5474#endif
5475
5476 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5477 return VINF_SUCCESS;
5478}
5479
5480
5481/**
5482 * Saves the guest control registers from the current VMCS into the guest-CPU
5483 * context.
5484 *
5485 * @returns VBox status code.
5486 * @param pVCpu Pointer to the VMCPU.
5487 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5488 * out-of-sync. Make sure to update the required fields
5489 * before using them.
5490 *
5491 * @remarks No-long-jump zone!!!
5492 */
5493static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5494{
5495 /* Guest CR0. Guest FPU. */
5496 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5497 AssertRCReturn(rc, rc);
5498
5499 /* Guest CR4. */
5500 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5501 AssertRCReturn(rc, rc);
5502
5503 /* Guest CR2 - updated always during the world-switch or in #PF. */
5504 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5505 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5506 {
5507 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5508 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5509
5510 PVM pVM = pVCpu->CTX_SUFF(pVM);
5511 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5512 || ( pVM->hm.s.fNestedPaging
5513 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5514 {
5515 uint64_t u64Val = 0;
5516 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5517 if (pMixedCtx->cr3 != u64Val)
5518 {
5519 CPUMSetGuestCR3(pVCpu, u64Val);
5520 if (VMMRZCallRing3IsEnabled(pVCpu))
5521 {
5522 PGMUpdateCR3(pVCpu, u64Val);
5523 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5524 }
5525 else
5526 {
5527 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5528 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5529 }
5530 }
5531
5532 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5533 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5534 {
5535 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5536 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5537 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5538 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5539
5540 if (VMMRZCallRing3IsEnabled(pVCpu))
5541 {
5542 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5543 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5544 }
5545 else
5546 {
5547 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5548 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5549 }
5550 }
5551 }
5552
5553 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5554 }
5555
5556 /*
5557 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5558 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5559 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5560 *
5561 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5562 */
5563 if (VMMRZCallRing3IsEnabled(pVCpu))
5564 {
5565 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5566 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5567
5568 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5569 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5570
5571 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5572 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5573 }
5574
5575 return rc;
5576}
5577
5578
5579/**
5580 * Reads a guest segment register from the current VMCS into the guest-CPU
5581 * context.
5582 *
5583 * @returns VBox status code.
5584 * @param pVCpu Pointer to the VMCPU.
5585 * @param idxSel Index of the selector in the VMCS.
5586 * @param idxLimit Index of the segment limit in the VMCS.
5587 * @param idxBase Index of the segment base in the VMCS.
5588 * @param idxAccess Index of the access rights of the segment in the VMCS.
5589 * @param pSelReg Pointer to the segment selector.
5590 *
5591 * @remarks No-long-jump zone!!!
5592 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5593 * macro as that takes care of whether to read from the VMCS cache or
5594 * not.
5595 */
5596DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5597 PCPUMSELREG pSelReg)
5598{
5599 uint32_t u32Val = 0;
5600 int rc = VMXReadVmcs32(idxSel, &u32Val);
5601 AssertRCReturn(rc, rc);
5602 pSelReg->Sel = (uint16_t)u32Val;
5603 pSelReg->ValidSel = (uint16_t)u32Val;
5604 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5605
5606 rc = VMXReadVmcs32(idxLimit, &u32Val);
5607 AssertRCReturn(rc, rc);
5608 pSelReg->u32Limit = u32Val;
5609
5610 uint64_t u64Val = 0;
5611 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5612 AssertRCReturn(rc, rc);
5613 pSelReg->u64Base = u64Val;
5614
5615 rc = VMXReadVmcs32(idxAccess, &u32Val);
5616 AssertRCReturn(rc, rc);
5617 pSelReg->Attr.u = u32Val;
5618
5619 /*
5620 * If VT-x marks the segment as unusable, most other bits remain undefined:
5621 * - For CS the L, D and G bits have meaning.
5622 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5623 * - For the remaining data segments no bits are defined.
5624 *
5625 * The present bit and the unusable bit has been observed to be set at the
5626 * same time (the selector was supposed to invalid as we started executing
5627 * a V8086 interrupt in ring-0).
5628 *
5629 * What should be important for the rest of the VBox code that the P bit is
5630 * cleared. Some of the other VBox code recognizes the unusable bit, but
5631 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5632 * safe side here, we'll strip off P and other bits we don't care about. If
5633 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5634 *
5635 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5636 */
5637 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5638 {
5639 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5640
5641 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5642 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5643 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5644
5645 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5646#ifdef DEBUG_bird
5647 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5648 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5649 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5650#endif
5651 }
5652 return VINF_SUCCESS;
5653}
5654
5655
5656#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5657# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5658 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5659 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5660#else
5661# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5662 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5663 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5664#endif
5665
5666
5667/**
5668 * Saves the guest segment registers from the current VMCS into the guest-CPU
5669 * context.
5670 *
5671 * @returns VBox status code.
5672 * @param pVCpu Pointer to the VMCPU.
5673 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5674 * out-of-sync. Make sure to update the required fields
5675 * before using them.
5676 *
5677 * @remarks No-long-jump zone!!!
5678 */
5679static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5680{
5681 /* Guest segment registers. */
5682 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5683 {
5684 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5685 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5686 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5687 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5688 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5689 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5690 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5691
5692 /* Restore segment attributes for real-on-v86 mode hack. */
5693 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5694 {
5695 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5696 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5697 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5698 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5699 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5700 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5701 }
5702 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5703 }
5704
5705 return VINF_SUCCESS;
5706}
5707
5708
5709/**
5710 * Saves the guest descriptor table registers and task register from the current
5711 * VMCS into the guest-CPU context.
5712 *
5713 * @returns VBox status code.
5714 * @param pVCpu Pointer to the VMCPU.
5715 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5716 * out-of-sync. Make sure to update the required fields
5717 * before using them.
5718 *
5719 * @remarks No-long-jump zone!!!
5720 */
5721static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5722{
5723 int rc = VINF_SUCCESS;
5724
5725 /* Guest LDTR. */
5726 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5727 {
5728 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5729 AssertRCReturn(rc, rc);
5730 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5731 }
5732
5733 /* Guest GDTR. */
5734 uint64_t u64Val = 0;
5735 uint32_t u32Val = 0;
5736 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5737 {
5738 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5739 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5740 pMixedCtx->gdtr.pGdt = u64Val;
5741 pMixedCtx->gdtr.cbGdt = u32Val;
5742 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
5743 }
5744
5745 /* Guest IDTR. */
5746 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5747 {
5748 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5749 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5750 pMixedCtx->idtr.pIdt = u64Val;
5751 pMixedCtx->idtr.cbIdt = u32Val;
5752 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
5753 }
5754
5755 /* Guest TR. */
5756 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5757 {
5758 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5759 AssertRCReturn(rc, rc);
5760
5761 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5762 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5763 {
5764 rc = VMXLOCAL_READ_SEG(TR, tr);
5765 AssertRCReturn(rc, rc);
5766 }
5767 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
5768 }
5769 return rc;
5770}
5771
5772#undef VMXLOCAL_READ_SEG
5773
5774
5775/**
5776 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5777 * context.
5778 *
5779 * @returns VBox status code.
5780 * @param pVCpu Pointer to the VMCPU.
5781 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5782 * out-of-sync. Make sure to update the required fields
5783 * before using them.
5784 *
5785 * @remarks No-long-jump zone!!!
5786 */
5787static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5788{
5789 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5790 {
5791 /** @todo We need to update DR7 according to what was done in hmR0VmxLoadSharedDebugState(). */
5792 if (!CPUMIsHyperDebugStateActive(pVCpu))
5793 {
5794 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5795 uint32_t u32Val;
5796 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5797 pMixedCtx->dr[7] = u32Val;
5798 }
5799
5800 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
5801 }
5802 return VINF_SUCCESS;
5803}
5804
5805
5806/**
5807 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
5808 *
5809 * @returns VBox status code.
5810 * @param pVCpu Pointer to the VMCPU.
5811 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5812 * out-of-sync. Make sure to update the required fields
5813 * before using them.
5814 *
5815 * @remarks No-long-jump zone!!!
5816 */
5817static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5818{
5819 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5820 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
5821 return VINF_SUCCESS;
5822}
5823
5824
5825/**
5826 * Saves the entire guest state from the currently active VMCS into the
5827 * guest-CPU context. This essentially VMREADs all guest-data.
5828 *
5829 * @returns VBox status code.
5830 * @param pVCpu Pointer to the VMCPU.
5831 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5832 * out-of-sync. Make sure to update the required fields
5833 * before using them.
5834 */
5835static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5836{
5837 Assert(pVCpu);
5838 Assert(pMixedCtx);
5839
5840 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5841 return VINF_SUCCESS;
5842
5843 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5844 again on the ring-3 callback path, there is no real need to. */
5845 if (VMMRZCallRing3IsEnabled(pVCpu))
5846 VMMR0LogFlushDisable(pVCpu);
5847 else
5848 Assert(VMMR0IsLogFlushDisabled(pVCpu));
5849 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5850
5851 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5852 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5853
5854 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5855 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5856
5857 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5858 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5859
5860 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5861 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5862
5863 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5864 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5865
5866 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5867 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5868
5869 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5870 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5871
5872 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5873 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5874
5875 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5876 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5877
5878 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5879 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5880
5881 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5882 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5883
5884 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5885 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5886
5887 if (VMMRZCallRing3IsEnabled(pVCpu))
5888 VMMR0LogFlushEnable(pVCpu);
5889
5890 return rc;
5891}
5892
5893
5894/**
5895 * Check per-VM and per-VCPU force flag actions that require us to go back to
5896 * ring-3 for one reason or another.
5897 *
5898 * @returns VBox status code (information status code included).
5899 * @retval VINF_SUCCESS if we don't have any actions that require going back to
5900 * ring-3.
5901 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5902 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5903 * interrupts)
5904 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5905 * all EMTs to be in ring-3.
5906 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5907 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5908 * to the EM loop.
5909 *
5910 * @param pVM Pointer to the VM.
5911 * @param pVCpu Pointer to the VMCPU.
5912 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5913 * out-of-sync. Make sure to update the required fields
5914 * before using them.
5915 */
5916static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5917{
5918 Assert(VMMRZCallRing3IsEnabled(pVCpu));
5919
5920 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5921 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5922 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5923 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5924 {
5925 /* We need the control registers now, make sure the guest-CPU context is updated. */
5926 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5927 AssertRCReturn(rc3, rc3);
5928
5929 /* Pending HM CR3 sync. */
5930 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5931 {
5932 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5933 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
5934 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5935 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5936 }
5937
5938 /* Pending HM PAE PDPEs. */
5939 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5940 {
5941 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5942 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5943 }
5944
5945 /* Pending PGM C3 sync. */
5946 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5947 {
5948 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
5949 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5950 if (rc2 != VINF_SUCCESS)
5951 {
5952 AssertRC(rc2);
5953 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5954 return rc2;
5955 }
5956 }
5957
5958 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5959 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5960 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5961 {
5962 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5963 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5964 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5965 return rc2;
5966 }
5967
5968 /* Pending VM request packets, such as hardware interrupts. */
5969 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5970 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5971 {
5972 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5973 return VINF_EM_PENDING_REQUEST;
5974 }
5975
5976 /* Pending PGM pool flushes. */
5977 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5978 {
5979 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5980 return VINF_PGM_POOL_FLUSH_PENDING;
5981 }
5982
5983 /* Pending DMA requests. */
5984 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5985 {
5986 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5987 return VINF_EM_RAW_TO_R3;
5988 }
5989 }
5990
5991 return VINF_SUCCESS;
5992}
5993
5994
5995/**
5996 * Converts any TRPM trap into a pending HM event. This is typically used when
5997 * entering from ring-3 (not longjmp returns).
5998 *
5999 * @param pVCpu Pointer to the VMCPU.
6000 */
6001static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6002{
6003 Assert(TRPMHasTrap(pVCpu));
6004 Assert(!pVCpu->hm.s.Event.fPending);
6005
6006 uint8_t uVector;
6007 TRPMEVENT enmTrpmEvent;
6008 RTGCUINT uErrCode;
6009 RTGCUINTPTR GCPtrFaultAddress;
6010 uint8_t cbInstr;
6011
6012 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6013 AssertRC(rc);
6014
6015 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
6016 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6017 if (enmTrpmEvent == TRPM_TRAP)
6018 {
6019 switch (uVector)
6020 {
6021 case X86_XCPT_BP:
6022 case X86_XCPT_OF:
6023 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6024 break;
6025
6026 case X86_XCPT_PF:
6027 case X86_XCPT_DF:
6028 case X86_XCPT_TS:
6029 case X86_XCPT_NP:
6030 case X86_XCPT_SS:
6031 case X86_XCPT_GP:
6032 case X86_XCPT_AC:
6033 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6034 /* no break! */
6035 default:
6036 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6037 break;
6038 }
6039 }
6040 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6041 {
6042 if (uVector == X86_XCPT_NMI)
6043 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6044 else
6045 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6046 }
6047 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6048 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6049 else
6050 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6051
6052 rc = TRPMResetTrap(pVCpu);
6053 AssertRC(rc);
6054 Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6055 u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6056
6057 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6058 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6059}
6060
6061
6062/**
6063 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6064 * VT-x to execute any instruction.
6065 *
6066 * @param pvCpu Pointer to the VMCPU.
6067 */
6068static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6069{
6070 Assert(pVCpu->hm.s.Event.fPending);
6071
6072 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6073 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
6074 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
6075 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6076
6077 /* If a trap was already pending, we did something wrong! */
6078 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6079
6080 TRPMEVENT enmTrapType;
6081 switch (uVectorType)
6082 {
6083 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6084 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6085 enmTrapType = TRPM_HARDWARE_INT;
6086 break;
6087
6088 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6089 enmTrapType = TRPM_SOFTWARE_INT;
6090 break;
6091
6092 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6093 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6094 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6095 enmTrapType = TRPM_TRAP;
6096 break;
6097
6098 default:
6099 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6100 enmTrapType = TRPM_32BIT_HACK;
6101 break;
6102 }
6103
6104 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6105
6106 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6107 AssertRC(rc);
6108
6109 if (fErrorCodeValid)
6110 TRPMSetErrorCode(pVCpu, uErrorCode);
6111
6112 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6113 && uVector == X86_XCPT_PF)
6114 {
6115 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6116 }
6117 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6118 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6119 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6120 {
6121 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6122 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6123 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6124 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6125 }
6126 pVCpu->hm.s.Event.fPending = false;
6127}
6128
6129
6130/**
6131 * Does the necessary state syncing before returning to ring-3 for any reason
6132 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6133 *
6134 * @returns VBox status code.
6135 * @param pVM Pointer to the VM.
6136 * @param pVCpu Pointer to the VMCPU.
6137 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6138 * be out-of-sync. Make sure to update the required
6139 * fields before using them.
6140 * @param fSaveGuestState Whether to save the guest state or not.
6141 *
6142 * @remarks No-long-jmp zone!!!
6143 */
6144static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6145{
6146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6147 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6148
6149 RTCPUID idCpu = RTMpCpuId();
6150 Log4Func(("HostCpuId=%u\n", idCpu));
6151
6152 /* Save the guest state if necessary. */
6153 if ( fSaveGuestState
6154 && pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6155 {
6156 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6157 AssertRCReturn(rc, rc);
6158 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6159 }
6160
6161 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6162 if (CPUMIsGuestFPUStateActive(pVCpu))
6163 {
6164 if (!fSaveGuestState)
6165 {
6166 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6167 AssertRCReturn(rc, rc);
6168 }
6169 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6170 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6171 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
6172 }
6173
6174 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6175#ifdef VBOX_STRICT
6176 if (CPUMIsHyperDebugStateActive(pVCpu))
6177 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6178#endif
6179 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6180 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
6181 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6182 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6183
6184#if HC_ARCH_BITS == 64
6185 /* Restore host-state bits that VT-x only restores partially. */
6186 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6187 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6188 {
6189 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6190 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6191 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6192 }
6193#endif
6194
6195 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6196 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6197 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6198 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6199 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6200 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6201 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6202 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6203
6204 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6205
6206 /** @todo This kinda defeats the purpose of having preemption hooks.
6207 * The problem is, deregistering the hooks should be moved to a place that
6208 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6209 * context.
6210 */
6211 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6212 {
6213 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6214 AssertRCReturn(rc, rc);
6215
6216 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6217 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6218 }
6219 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6220 NOREF(idCpu);
6221
6222 return VINF_SUCCESS;
6223}
6224
6225
6226/**
6227 * Leaves the VT-x session.
6228 *
6229 * @returns VBox status code.
6230 * @param pVM Pointer to the VM.
6231 * @param pVCpu Pointer to the VMCPU.
6232 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6233 * out-of-sync. Make sure to update the required fields
6234 * before using them.
6235 *
6236 * @remarks No-long-jmp zone!!!
6237 */
6238DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6239{
6240 HM_DISABLE_PREEMPT_IF_NEEDED();
6241 HMVMX_ASSERT_CPU_SAFE();
6242 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6243 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6244
6245 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6246 and done this from the VMXR0ThreadCtxCallback(). */
6247 if (!pVCpu->hm.s.fLeaveDone)
6248 {
6249 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6250 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6251 pVCpu->hm.s.fLeaveDone = true;
6252 }
6253
6254 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6255 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6256 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6257 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6258 VMMR0ThreadCtxHooksDeregister(pVCpu);
6259
6260 /* Leave HM context. This takes care of local init (term). */
6261 int rc = HMR0LeaveCpu(pVCpu);
6262
6263 HM_RESTORE_PREEMPT_IF_NEEDED();
6264
6265 return rc;
6266}
6267
6268
6269/**
6270 * Does the necessary state syncing before doing a longjmp to ring-3.
6271 *
6272 * @returns VBox status code.
6273 * @param pVM Pointer to the VM.
6274 * @param pVCpu Pointer to the VMCPU.
6275 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6276 * out-of-sync. Make sure to update the required fields
6277 * before using them.
6278 *
6279 * @remarks No-long-jmp zone!!!
6280 */
6281DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6282{
6283 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6284}
6285
6286
6287/**
6288 * Take necessary actions before going back to ring-3.
6289 *
6290 * An action requires us to go back to ring-3. This function does the necessary
6291 * steps before we can safely return to ring-3. This is not the same as longjmps
6292 * to ring-3, this is voluntary and prepares the guest so it may continue
6293 * executing outside HM (recompiler/IEM).
6294 *
6295 * @returns VBox status code.
6296 * @param pVM Pointer to the VM.
6297 * @param pVCpu Pointer to the VMCPU.
6298 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6299 * out-of-sync. Make sure to update the required fields
6300 * before using them.
6301 * @param rcExit The reason for exiting to ring-3. Can be
6302 * VINF_VMM_UNKNOWN_RING3_CALL.
6303 */
6304static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6305{
6306 Assert(pVM);
6307 Assert(pVCpu);
6308 Assert(pMixedCtx);
6309 HMVMX_ASSERT_PREEMPT_SAFE();
6310
6311 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6312 {
6313 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6314 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6315 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6316 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6317 }
6318
6319 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6320 VMMRZCallRing3Disable(pVCpu);
6321 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6322
6323 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6324 if (pVCpu->hm.s.Event.fPending)
6325 {
6326 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6327 Assert(!pVCpu->hm.s.Event.fPending);
6328 }
6329
6330 /* Save guest state and restore host state bits. */
6331 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6332 AssertRCReturn(rc, rc);
6333 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6334
6335 /* Sync recompiler state. */
6336 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6337 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6338 | CPUM_CHANGED_LDTR
6339 | CPUM_CHANGED_GDTR
6340 | CPUM_CHANGED_IDTR
6341 | CPUM_CHANGED_TR
6342 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6343 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6344 if ( pVM->hm.s.fNestedPaging
6345 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6346 {
6347 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6348 }
6349
6350 /*
6351 * Clear the X86_EFL_TF if necessary.
6352 */
6353 if (pVCpu->hm.s.fClearTrapFlag)
6354 {
6355 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6356 pMixedCtx->eflags.Bits.u1TF = 0;
6357 pVCpu->hm.s.fClearTrapFlag = false;
6358 }
6359 /** @todo there seems to be issues with the resume flag when the monitor trap
6360 * flag is pending without being used. Seen early in bios init when
6361 * accessing APIC page in prot mode. */
6362
6363 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6364 if (rcExit != VINF_EM_RAW_INTERRUPT)
6365 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
6366
6367 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6368
6369 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6370 VMMRZCallRing3RemoveNotification(pVCpu);
6371 VMMRZCallRing3Enable(pVCpu);
6372
6373 return rc;
6374}
6375
6376
6377/**
6378 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6379 * longjump to ring-3 and possibly get preempted.
6380 *
6381 * @returns VBox status code.
6382 * @param pVCpu Pointer to the VMCPU.
6383 * @param enmOperation The operation causing the ring-3 longjump.
6384 * @param pvUser Opaque pointer to the guest-CPU context. The data
6385 * may be out-of-sync. Make sure to update the required
6386 * fields before using them.
6387 */
6388DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6389{
6390 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6391 {
6392 VMMRZCallRing3RemoveNotification(pVCpu);
6393 HM_DISABLE_PREEMPT_IF_NEEDED();
6394
6395 /* If anything here asserts or fails, good luck. */
6396 if (CPUMIsGuestFPUStateActive(pVCpu))
6397 CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6398
6399 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6400
6401#if HC_ARCH_BITS == 64
6402 /* Restore host-state bits that VT-x only restores partially. */
6403 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6404 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6405 {
6406 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6407 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6408 }
6409#endif
6410 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6411 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6412 {
6413 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6414 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6415 }
6416
6417 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6418 VMMR0ThreadCtxHooksDeregister(pVCpu);
6419
6420 HMR0LeaveCpu(pVCpu);
6421 HM_RESTORE_PREEMPT_IF_NEEDED();
6422 return VINF_SUCCESS;
6423 }
6424
6425 Assert(pVCpu);
6426 Assert(pvUser);
6427 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6428 HMVMX_ASSERT_PREEMPT_SAFE();
6429
6430 VMMRZCallRing3Disable(pVCpu);
6431 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6432
6433 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
6434 enmOperation));
6435
6436 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6437 AssertRCReturn(rc, rc);
6438
6439 VMMRZCallRing3Enable(pVCpu);
6440 return VINF_SUCCESS;
6441}
6442
6443
6444/**
6445 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6446 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6447 *
6448 * @param pVCpu Pointer to the VMCPU.
6449 */
6450DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6451{
6452 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6453 {
6454 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6455 {
6456 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6457 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6458 AssertRC(rc);
6459 }
6460 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6461}
6462
6463
6464/**
6465 * Evaluates the event to be delivered to the guest and sets it as the pending
6466 * event.
6467 *
6468 * @param pVCpu Pointer to the VMCPU.
6469 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6470 * out-of-sync. Make sure to update the required fields
6471 * before using them.
6472 */
6473static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6474{
6475 Assert(!pVCpu->hm.s.Event.fPending);
6476
6477 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6478 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6479 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6480 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6481
6482 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6483 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6484 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6485 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6486 Assert(!TRPMHasTrap(pVCpu));
6487
6488 /** @todo SMI. SMIs take priority over NMIs. */
6489 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6490 {
6491 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6492 if ( !fBlockMovSS
6493 && !fBlockSti)
6494 {
6495 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6496 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6497 uint32_t u32IntrInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6498 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6499
6500 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6501 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6502 }
6503 else
6504 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6505 }
6506 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6507 && !pVCpu->hm.s.fSingleInstruction)
6508 {
6509 /*
6510 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6511 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6512 * evaluated here and not set as pending, solely based on the force-flags.
6513 */
6514 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6515 AssertRC(rc);
6516 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6517 if ( !fBlockInt
6518 && !fBlockSti
6519 && !fBlockMovSS)
6520 {
6521 uint8_t u8Interrupt;
6522 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6523 if (RT_SUCCESS(rc))
6524 {
6525 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6526 uint32_t u32IntrInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6527 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6528
6529 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6530 }
6531 else
6532 {
6533 /** @todo Does this actually happen? If not turn it into an assertion. */
6534 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6535 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6536 }
6537 }
6538 else
6539 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6540 }
6541}
6542
6543
6544/**
6545 * Injects any pending events into the guest if the guest is in a state to
6546 * receive them.
6547 *
6548 * @returns VBox status code (informational status codes included).
6549 * @param pVCpu Pointer to the VMCPU.
6550 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6551 * out-of-sync. Make sure to update the required fields
6552 * before using them.
6553 */
6554static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6555{
6556 HMVMX_ASSERT_PREEMPT_SAFE();
6557 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6558
6559 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6560 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6561 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6562 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6563
6564 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6565 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6566 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6567 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6568 Assert(!TRPMHasTrap(pVCpu));
6569
6570 int rc = VINF_SUCCESS;
6571 if (pVCpu->hm.s.Event.fPending)
6572 {
6573#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6574 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6575 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6576 {
6577 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6578 AssertRCReturn(rc, rc);
6579 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6580 Assert(!fBlockInt);
6581 Assert(!fBlockSti);
6582 Assert(!fBlockMovSS);
6583 }
6584 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6585 {
6586 Assert(!fBlockSti);
6587 Assert(!fBlockMovSS);
6588 }
6589#endif
6590 Log4(("Injecting pending event vcpu[%RU32] u64IntrInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo));
6591 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6592 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6593 AssertRCReturn(rc, rc);
6594
6595 /* Update the interruptibility-state as it could have been changed by
6596 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6597 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6598 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6599
6600#ifdef VBOX_WITH_STATISTICS
6601 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6602 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6603 else
6604 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6605#endif
6606 }
6607
6608 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6609 int rc2 = VINF_SUCCESS;
6610 if ( fBlockSti
6611 || fBlockMovSS)
6612 {
6613 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6614 {
6615 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6616 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6617 {
6618 /*
6619 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6620 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6621 * See Intel spec. 27.3.4 "Saving Non-Register State".
6622 */
6623 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6624 AssertRCReturn(rc, rc);
6625 }
6626 }
6627 else
6628 {
6629 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6630 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6631 uIntrState = 0;
6632 }
6633 }
6634
6635 /*
6636 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6637 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6638 */
6639 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6640 AssertRC(rc2);
6641
6642 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6643 return rc;
6644}
6645
6646
6647/**
6648 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6649 *
6650 * @param pVCpu Pointer to the VMCPU.
6651 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6652 * out-of-sync. Make sure to update the required fields
6653 * before using them.
6654 */
6655DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6656{
6657 uint32_t u32IntrInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6658 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6659}
6660
6661
6662/**
6663 * Injects a double-fault (#DF) exception into the VM.
6664 *
6665 * @returns VBox status code (informational status code included).
6666 * @param pVCpu Pointer to the VMCPU.
6667 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6668 * out-of-sync. Make sure to update the required fields
6669 * before using them.
6670 */
6671DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6672{
6673 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6674 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6675 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6676 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6677 puIntrState);
6678}
6679
6680
6681/**
6682 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6683 *
6684 * @param pVCpu Pointer to the VMCPU.
6685 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6686 * out-of-sync. Make sure to update the required fields
6687 * before using them.
6688 */
6689DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6690{
6691 uint32_t u32IntrInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6692 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6693 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6694}
6695
6696
6697/**
6698 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6699 *
6700 * @param pVCpu Pointer to the VMCPU.
6701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6702 * out-of-sync. Make sure to update the required fields
6703 * before using them.
6704 * @param cbInstr The value of RIP that is to be pushed on the guest
6705 * stack.
6706 */
6707DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6708{
6709 uint32_t u32IntrInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6710 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6711 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6712}
6713
6714
6715/**
6716 * Injects a general-protection (#GP) fault into the VM.
6717 *
6718 * @returns VBox status code (informational status code included).
6719 * @param pVCpu Pointer to the VMCPU.
6720 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6721 * out-of-sync. Make sure to update the required fields
6722 * before using them.
6723 * @param u32ErrorCode The error code associated with the #GP.
6724 */
6725DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6726 uint32_t *puIntrState)
6727{
6728 uint32_t u32IntrInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6729 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6730 if (fErrorCodeValid)
6731 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6732 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6733 puIntrState);
6734}
6735
6736
6737/**
6738 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6739 *
6740 * @param pVCpu Pointer to the VMCPU.
6741 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6742 * out-of-sync. Make sure to update the required fields
6743 * before using them.
6744 * @param uVector The software interrupt vector number.
6745 * @param cbInstr The value of RIP that is to be pushed on the guest
6746 * stack.
6747 */
6748DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6749{
6750 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6751 if ( uVector == X86_XCPT_BP
6752 || uVector == X86_XCPT_OF)
6753 {
6754 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6755 }
6756 else
6757 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6758 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6759}
6760
6761
6762/**
6763 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6764 * stack.
6765 *
6766 * @returns VBox status code (information status code included).
6767 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6768 * @param pVM Pointer to the VM.
6769 * @param pMixedCtx Pointer to the guest-CPU context.
6770 * @param uValue The value to push to the guest stack.
6771 */
6772DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6773{
6774 /*
6775 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6776 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6777 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6778 */
6779 if (pMixedCtx->sp == 1)
6780 return VINF_EM_RESET;
6781 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6782 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6783 AssertRCReturn(rc, rc);
6784 return rc;
6785}
6786
6787
6788/**
6789 * Injects an event into the guest upon VM-entry by updating the relevant fields
6790 * in the VM-entry area in the VMCS.
6791 *
6792 * @returns VBox status code (informational error codes included).
6793 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6794 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6795 *
6796 * @param pVCpu Pointer to the VMCPU.
6797 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6798 * be out-of-sync. Make sure to update the required
6799 * fields before using them.
6800 * @param u64IntrInfo The VM-entry interruption-information field.
6801 * @param cbInstr The VM-entry instruction length in bytes (for
6802 * software interrupts, exceptions and privileged
6803 * software exceptions).
6804 * @param u32ErrCode The VM-entry exception error code.
6805 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6806 * @param puIntrState Pointer to the current guest interruptibility-state.
6807 * This interruptibility-state will be updated if
6808 * necessary. This cannot not be NULL.
6809 *
6810 * @remarks Requires CR0!
6811 * @remarks No-long-jump zone!!!
6812 */
6813static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6814 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6815{
6816 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6817 AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6818 Assert(puIntrState);
6819 uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6820
6821 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6822 const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6823
6824#ifdef VBOX_STRICT
6825 /* Validate the error-code-valid bit for hardware exceptions. */
6826 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6827 {
6828 switch (uVector)
6829 {
6830 case X86_XCPT_PF:
6831 case X86_XCPT_DF:
6832 case X86_XCPT_TS:
6833 case X86_XCPT_NP:
6834 case X86_XCPT_SS:
6835 case X86_XCPT_GP:
6836 case X86_XCPT_AC:
6837 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6838 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6839 /* fallthru */
6840 default:
6841 break;
6842 }
6843 }
6844#endif
6845
6846 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
6847 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6848 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6849
6850 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6851
6852 /* We require CR0 to check if the guest is in real-mode. */
6853 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6854 AssertRCReturn(rc, rc);
6855
6856 /*
6857 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6858 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6859 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6860 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6861 */
6862 if (CPUMIsGuestInRealModeEx(pMixedCtx))
6863 {
6864 PVM pVM = pVCpu->CTX_SUFF(pVM);
6865 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6866 {
6867 Assert(PDMVmmDevHeapIsEnabled(pVM));
6868 Assert(pVM->hm.s.vmx.pRealModeTSS);
6869
6870 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6871 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6872 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6873 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6874 AssertRCReturn(rc, rc);
6875 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6876
6877 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6878 const size_t cbIdtEntry = sizeof(X86IDTR16);
6879 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6880 {
6881 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6882 if (uVector == X86_XCPT_DF)
6883 return VINF_EM_RESET;
6884 else if (uVector == X86_XCPT_GP)
6885 {
6886 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6887 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6888 }
6889
6890 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6891 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6892 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
6893 }
6894
6895 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6896 uint16_t uGuestIp = pMixedCtx->ip;
6897 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6898 {
6899 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
6900 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
6901 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6902 }
6903 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6904 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6905
6906 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6907 X86IDTR16 IdtEntry;
6908 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6909 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6910 AssertRCReturn(rc, rc);
6911
6912 /* Construct the stack frame for the interrupt/exception handler. */
6913 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6914 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6915 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6916 AssertRCReturn(rc, rc);
6917
6918 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
6919 if (rc == VINF_SUCCESS)
6920 {
6921 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
6922 pMixedCtx->rip = IdtEntry.offSel;
6923 pMixedCtx->cs.Sel = IdtEntry.uSel;
6924 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6925 if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6926 && uVector == X86_XCPT_PF)
6927 {
6928 pMixedCtx->cr2 = GCPtrFaultAddress;
6929 }
6930
6931 /* If any other guest-state bits are changed here, make sure to update
6932 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6933 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS
6934 | HM_CHANGED_GUEST_RIP
6935 | HM_CHANGED_GUEST_RFLAGS
6936 | HM_CHANGED_GUEST_RSP;
6937
6938 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6939 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6940 {
6941 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6942 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6943 Log4(("Clearing inhibition due to STI.\n"));
6944 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6945 }
6946 Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6947
6948 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
6949 it, if we are returning to ring-3 before executing guest code. */
6950 pVCpu->hm.s.Event.fPending = false;
6951 }
6952 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6953 return rc;
6954 }
6955 else
6956 {
6957 /*
6958 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6959 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6960 */
6961 u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6962 }
6963 }
6964
6965 /* Validate. */
6966 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6967 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6968 Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6969
6970 /* Inject. */
6971 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6972 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6973 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6974 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6975
6976 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6977 && uVector == X86_XCPT_PF)
6978 {
6979 pMixedCtx->cr2 = GCPtrFaultAddress;
6980 }
6981
6982 Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6983 u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6984
6985 AssertRCReturn(rc, rc);
6986 return rc;
6987}
6988
6989
6990/**
6991 * Clears the interrupt-window exiting control in the VMCS and if necessary
6992 * clears the current event in the VMCS as well.
6993 *
6994 * @returns VBox status code.
6995 * @param pVCpu Pointer to the VMCPU.
6996 *
6997 * @remarks Use this function only to clear events that have not yet been
6998 * delivered to the guest but are injected in the VMCS!
6999 * @remarks No-long-jump zone!!!
7000 */
7001static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7002{
7003 int rc;
7004 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7005
7006 /* Clear interrupt-window exiting control. */
7007 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7008 {
7009 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7010 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7011 AssertRC(rc);
7012 }
7013
7014 if (!pVCpu->hm.s.Event.fPending)
7015 return;
7016
7017#ifdef VBOX_STRICT
7018 uint32_t u32EntryInfo;
7019 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7020 AssertRC(rc);
7021 Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
7022#endif
7023
7024 /* Clear the entry-interruption field (including the valid bit). */
7025 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7026 AssertRC(rc);
7027
7028 /* Clear the pending debug exception field. */
7029 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7030 AssertRC(rc);
7031}
7032
7033
7034/**
7035 * Enters the VT-x session.
7036 *
7037 * @returns VBox status code.
7038 * @param pVM Pointer to the VM.
7039 * @param pVCpu Pointer to the VMCPU.
7040 * @param pCpu Pointer to the CPU info struct.
7041 */
7042VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7043{
7044 AssertPtr(pVM);
7045 AssertPtr(pVCpu);
7046 Assert(pVM->hm.s.vmx.fSupported);
7047 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7048 NOREF(pCpu);
7049
7050 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7051 Assert((pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE))
7052 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7053
7054#ifdef VBOX_STRICT
7055 /* Make sure we're in VMX root mode. */
7056 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7057 if (!(u32HostCR4 & X86_CR4_VMXE))
7058 {
7059 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7060 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7061 }
7062#endif
7063
7064 /*
7065 * Load the VCPU's VMCS as the current (and active) one.
7066 */
7067 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7068 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7069 if (RT_FAILURE(rc))
7070 return rc;
7071
7072 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7073 pVCpu->hm.s.fLeaveDone = false;
7074 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7075
7076 return VINF_SUCCESS;
7077}
7078
7079
7080/**
7081 * The thread-context callback (only on platforms which support it).
7082 *
7083 * @param enmEvent The thread-context event.
7084 * @param pVCpu Pointer to the VMCPU.
7085 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7086 * @thread EMT.
7087 */
7088VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7089{
7090 switch (enmEvent)
7091 {
7092 case RTTHREADCTXEVENT_PREEMPTING:
7093 {
7094 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7095 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7096 VMCPU_ASSERT_EMT(pVCpu);
7097
7098 PVM pVM = pVCpu->CTX_SUFF(pVM);
7099 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7100
7101 /* No longjmps (logger flushes, locks) in this fragile context. */
7102 VMMRZCallRing3Disable(pVCpu);
7103 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7104
7105 /*
7106 * Restore host-state (FPU, debug etc.)
7107 */
7108 if (!pVCpu->hm.s.fLeaveDone)
7109 {
7110 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7111 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7112 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7113 pVCpu->hm.s.fLeaveDone = true;
7114 }
7115
7116 /* Leave HM context, takes care of local init (term). */
7117 int rc = HMR0LeaveCpu(pVCpu);
7118 AssertRC(rc); NOREF(rc);
7119
7120 /* Restore longjmp state. */
7121 VMMRZCallRing3Enable(pVCpu);
7122 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7123 break;
7124 }
7125
7126 case RTTHREADCTXEVENT_RESUMED:
7127 {
7128 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7129 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7130 VMCPU_ASSERT_EMT(pVCpu);
7131
7132 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7133 VMMRZCallRing3Disable(pVCpu);
7134 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7135
7136 /* Initialize the bare minimum state required for HM. This takes care of
7137 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7138 int rc = HMR0EnterCpu(pVCpu);
7139 AssertRC(rc);
7140 Assert((pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE))
7141 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7142
7143 /* Load the active VMCS as the current one. */
7144 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7145 {
7146 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7147 AssertRC(rc); NOREF(rc);
7148 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7149 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7150 }
7151 pVCpu->hm.s.fLeaveDone = false;
7152
7153 /* Restore longjmp state. */
7154 VMMRZCallRing3Enable(pVCpu);
7155 break;
7156 }
7157
7158 default:
7159 break;
7160 }
7161}
7162
7163
7164/**
7165 * Saves the host state in the VMCS host-state.
7166 * Sets up the VM-exit MSR-load area.
7167 *
7168 * The CPU state will be loaded from these fields on every successful VM-exit.
7169 *
7170 * @returns VBox status code.
7171 * @param pVM Pointer to the VM.
7172 * @param pVCpu Pointer to the VMCPU.
7173 *
7174 * @remarks No-long-jump zone!!!
7175 */
7176static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7177{
7178 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7179
7180 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
7181 return VINF_SUCCESS;
7182
7183 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7184 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7185
7186 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7187 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7188
7189 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7190 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7191
7192 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
7193 return rc;
7194}
7195
7196
7197/**
7198 * Saves the host state in the VMCS host-state.
7199 *
7200 * @returns VBox status code.
7201 * @param pVM Pointer to the VM.
7202 * @param pVCpu Pointer to the VMCPU.
7203 *
7204 * @remarks No-long-jump zone!!!
7205 */
7206VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7207{
7208 AssertPtr(pVM);
7209 AssertPtr(pVCpu);
7210
7211 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7212
7213 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7214 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7215 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7216 return hmR0VmxSaveHostState(pVM, pVCpu);
7217}
7218
7219
7220/**
7221 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7222 * loaded from these fields on every successful VM-entry.
7223 *
7224 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7225 * Sets up the VM-entry controls.
7226 * Sets up the appropriate VMX non-root function to execute guest code based on
7227 * the guest CPU mode.
7228 *
7229 * @returns VBox status code.
7230 * @param pVM Pointer to the VM.
7231 * @param pVCpu Pointer to the VMCPU.
7232 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7233 * out-of-sync. Make sure to update the required fields
7234 * before using them.
7235 *
7236 * @remarks No-long-jump zone!!!
7237 */
7238static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7239{
7240 AssertPtr(pVM);
7241 AssertPtr(pVCpu);
7242 AssertPtr(pMixedCtx);
7243 HMVMX_ASSERT_PREEMPT_SAFE();
7244
7245#ifdef LOG_ENABLED
7246 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7247 * probably not initialized yet? Anyway this will do for now.
7248 *
7249 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7250 * interface and disable ring-3 calls when thread-context hooks are not
7251 * available. */
7252 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7253 VMMR0LogFlushDisable(pVCpu);
7254#endif
7255
7256 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7257
7258 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7259
7260 /* Determine real-on-v86 mode. */
7261 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7262 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7263 && CPUMIsGuestInRealModeEx(pMixedCtx))
7264 {
7265 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7266 }
7267
7268 /*
7269 * Load the guest-state into the VMCS.
7270 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7271 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7272 */
7273 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7274 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7275
7276 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7277 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7278 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7279
7280 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7281 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7282 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7283
7284 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7285 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7286
7287 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7288 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7289
7290 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7291 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7292 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7293
7294 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7295 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7296
7297 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7298 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7299
7300 /*
7301 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7302 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7303 */
7304 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7305 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7306
7307 /* Clear any unused and reserved bits. */
7308 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
7309
7310#ifdef LOG_ENABLED
7311 /* Only reenable log-flushing if the caller has it enabled. */
7312 if (!fCallerDisabledLogFlush)
7313 VMMR0LogFlushEnable(pVCpu);
7314#endif
7315
7316 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7317 return rc;
7318}
7319
7320
7321/**
7322 * Loads the state shared between the host and guest into the VMCS.
7323 *
7324 * @param pVM Pointer to the VM.
7325 * @param pVCpu Pointer to the VMCPU.
7326 * @param pCtx Pointer to the guest-CPU context.
7327 *
7328 * @remarks No-long-jump zone!!!
7329 */
7330static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7331{
7332 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7333 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7334
7335 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
7336 {
7337 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7338 AssertRC(rc);
7339 }
7340
7341 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
7342 {
7343 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7344 AssertRC(rc);
7345
7346 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7347 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
7348 {
7349 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7350 AssertRC(rc);
7351 }
7352 }
7353
7354 AssertMsg(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE), ("fContextUseFlags=%#x\n",
7355 pVCpu->hm.s.fContextUseFlags));
7356}
7357
7358
7359/**
7360 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7361 *
7362 * @param pVM Pointer to the VM.
7363 * @param pVCpu Pointer to the VMCPU.
7364 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7365 * out-of-sync. Make sure to update the required fields
7366 * before using them.
7367 */
7368DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7369{
7370 HMVMX_ASSERT_PREEMPT_SAFE();
7371
7372 Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7373#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7374 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
7375#endif
7376
7377 if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7378 {
7379 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7380 AssertRC(rc);
7381 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7382 }
7383 else if (pVCpu->hm.s.fContextUseFlags)
7384 {
7385 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7386 AssertRC(rc);
7387 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7388 }
7389
7390 /* All the guest state bits should be loaded except maybe the host context and shared host/guest bits. */
7391 AssertMsg( !(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST)
7392 || !(pVCpu->hm.s.fContextUseFlags & ~(HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE)),
7393 ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7394
7395#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7396 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7397 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7398 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7399#endif
7400}
7401
7402
7403/**
7404 * Does the preparations before executing guest code in VT-x.
7405 *
7406 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7407 * recompiler. We must be cautious what we do here regarding committing
7408 * guest-state information into the VMCS assuming we assuredly execute the
7409 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7410 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7411 * so that the recompiler can (and should) use them when it resumes guest
7412 * execution. Otherwise such operations must be done when we can no longer
7413 * exit to ring-3.
7414 *
7415 * @returns Strict VBox status code.
7416 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7417 * have been disabled.
7418 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7419 * double-fault into the guest.
7420 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7421 *
7422 * @param pVM Pointer to the VM.
7423 * @param pVCpu Pointer to the VMCPU.
7424 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7425 * out-of-sync. Make sure to update the required fields
7426 * before using them.
7427 * @param pVmxTransient Pointer to the VMX transient structure.
7428 *
7429 * @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7430 * interrupts will be disabled.
7431 */
7432static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7433{
7434 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7435
7436#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7437 PGMRZDynMapFlushAutoSet(pVCpu);
7438#endif
7439
7440 /* Check force flag actions that might require us to go back to ring-3. */
7441 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7442 if (rc != VINF_SUCCESS)
7443 return rc;
7444
7445#ifndef IEM_VERIFICATION_MODE_FULL
7446 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7447 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7448 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7449 {
7450 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7451 RTGCPHYS GCPhysApicBase;
7452 GCPhysApicBase = pMixedCtx->msrApicBase;
7453 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7454
7455 /* Unalias any existing mapping. */
7456 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7457 AssertRCReturn(rc, rc);
7458
7459 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7460 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7461 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7462 AssertRCReturn(rc, rc);
7463
7464 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7465 }
7466#endif /* !IEM_VERIFICATION_MODE_FULL */
7467
7468 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7469 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7470
7471 /*
7472 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7473 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7474 */
7475 if (TRPMHasTrap(pVCpu))
7476 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7477 else if (!pVCpu->hm.s.Event.fPending)
7478 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7479
7480 /*
7481 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7482 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7483 */
7484 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7485 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7486 {
7487 Assert(rc == VINF_EM_RESET);
7488 return rc;
7489 }
7490
7491 /*
7492 * No longjmps to ring-3 from this point on!!!
7493 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7494 * This also disables flushing of the R0-logger instance (if any).
7495 */
7496 VMMRZCallRing3Disable(pVCpu);
7497
7498 /*
7499 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7500 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7501 *
7502 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7503 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7504 *
7505 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7506 * executing guest code.
7507 */
7508 pVmxTransient->uEflags = ASMIntDisableFlags();
7509 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7510 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7511 {
7512 hmR0VmxClearEventVmcs(pVCpu);
7513 ASMSetFlags(pVmxTransient->uEflags);
7514 VMMRZCallRing3Enable(pVCpu);
7515 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7516 return VINF_EM_RAW_TO_R3;
7517 }
7518 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7519 {
7520 hmR0VmxClearEventVmcs(pVCpu);
7521 ASMSetFlags(pVmxTransient->uEflags);
7522 VMMRZCallRing3Enable(pVCpu);
7523 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7524 return VINF_EM_RAW_INTERRUPT;
7525 }
7526
7527 /* We've injected any pending events. This is really the point of no return (to ring-3). */
7528 pVCpu->hm.s.Event.fPending = false;
7529
7530 return VINF_SUCCESS;
7531}
7532
7533
7534/**
7535 * Prepares to run guest code in VT-x and we've committed to doing so. This
7536 * means there is no backing out to ring-3 or anywhere else at this
7537 * point.
7538 *
7539 * @param pVM Pointer to the VM.
7540 * @param pVCpu Pointer to the VMCPU.
7541 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7542 * out-of-sync. Make sure to update the required fields
7543 * before using them.
7544 * @param pVmxTransient Pointer to the VMX transient structure.
7545 *
7546 * @remarks Called with preemption disabled.
7547 * @remarks No-long-jump zone!!!
7548 */
7549static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7550{
7551 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7552 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7553 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7554
7555 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7556 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7557
7558 /*
7559 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7560 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7561 * Reload only the necessary state, the assertion will catch if other parts of the code
7562 * change.
7563 */
7564 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7565 {
7566 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7567 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7568 }
7569
7570#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7571 if (!CPUMIsGuestFPUStateActive(pVCpu))
7572 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7573 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
7574#endif
7575
7576 /*
7577 * Load the host state bits as we may've been preempted (only happens when
7578 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7579 */
7580 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
7581 {
7582 /* This ASSUMES that pfnStartVM has been set up already. */
7583 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7584 AssertRC(rc);
7585 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7586 }
7587 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7588
7589 /*
7590 * Load the state shared between host and guest (FPU, debug).
7591 */
7592 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE)
7593 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7594 AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7595
7596 /* Store status of the shared guest-host state at the time of VM-entry. */
7597#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7598 if (CPUMIsGuestInLongModeEx(pMixedCtx))
7599 {
7600 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
7601 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
7602 }
7603 else
7604#endif
7605 {
7606 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
7607 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
7608 }
7609 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
7610
7611 /*
7612 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7613 */
7614 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7615 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7616
7617 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7618 RTCPUID idCurrentCpu = pCpu->idCpu;
7619 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7620 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
7621 {
7622 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7623 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7624 }
7625
7626 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7627 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7628 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7629 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7630
7631 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7632
7633 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7634 to start executing. */
7635
7636#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7637 /*
7638 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7639 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7640 */
7641 if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7642 && !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7643 {
7644 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7645 uint64_t u64HostTscAux = 0;
7646 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7647 AssertRC(rc2);
7648 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7649 }
7650#endif
7651}
7652
7653
7654/**
7655 * Performs some essential restoration of state after running guest code in
7656 * VT-x.
7657 *
7658 * @param pVM Pointer to the VM.
7659 * @param pVCpu Pointer to the VMCPU.
7660 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7661 * out-of-sync. Make sure to update the required fields
7662 * before using them.
7663 * @param pVmxTransient Pointer to the VMX transient structure.
7664 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7665 *
7666 * @remarks Called with interrupts disabled, and returns with interrups enabled!
7667 *
7668 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7669 * unconditionally when it is safe to do so.
7670 */
7671static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7672{
7673 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7674
7675 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7676 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7677 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7678 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7679 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7680
7681 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7682 {
7683#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7684 /* Restore host's TSC_AUX. */
7685 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7686 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7687#endif
7688 /** @todo Find a way to fix hardcoding a guestimate. */
7689 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7690 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7691 }
7692
7693 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7694 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7695 Assert(!(ASMGetFlags() & X86_EFL_IF));
7696 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7697
7698#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7699 if (CPUMIsGuestFPUStateActive(pVCpu))
7700 {
7701 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7702 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7703 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
7704 }
7705#endif
7706
7707 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7708 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7709 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7710 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7711
7712 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7713 uint32_t uExitReason;
7714 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7715 rc |= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7716 AssertRC(rc);
7717 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7718 pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7719
7720 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7721 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7722 {
7723 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7724 pVmxTransient->fVMEntryFailed));
7725 return;
7726 }
7727
7728 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7729 {
7730 /* Update the guest interruptibility-state from the VMCS. */
7731 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7732#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7733 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7734 AssertRC(rc);
7735#endif
7736 /*
7737 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7738 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7739 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7740 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7741 */
7742 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7743 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7744 {
7745 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7746 AssertRC(rc);
7747 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
7748 }
7749 }
7750}
7751
7752
7753
7754/**
7755 * Runs the guest code using VT-x the normal way.
7756 *
7757 * @returns VBox status code.
7758 * @param pVM Pointer to the VM.
7759 * @param pVCpu Pointer to the VMCPU.
7760 * @param pCtx Pointer to the guest-CPU context.
7761 *
7762 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
7763 * @remarks Called with preemption disabled.
7764 */
7765static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7766{
7767 VMXTRANSIENT VmxTransient;
7768 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7769 int rc = VERR_INTERNAL_ERROR_5;
7770 uint32_t cLoops = 0;
7771
7772 for (;; cLoops++)
7773 {
7774 Assert(!HMR0SuspendPending());
7775 HMVMX_ASSERT_CPU_SAFE();
7776
7777 /* Preparatory work for running guest code, this may force us to return
7778 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7779 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7780 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7781 if (rc != VINF_SUCCESS)
7782 break;
7783
7784 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7785 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7786 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7787
7788 /* Restore any residual host-state and save any bits shared between host
7789 and guest into the guest-CPU state. Re-enables interrupts! */
7790 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7791
7792 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7793 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7794 {
7795 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7796 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7797 return rc;
7798 }
7799
7800 /* Handle the VM-exit. */
7801 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7802 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7803 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7804 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7805 HMVMX_START_EXIT_DISPATCH_PROF();
7806#ifdef HMVMX_USE_FUNCTION_TABLE
7807 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7808#else
7809 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7810#endif
7811 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7812 if (rc != VINF_SUCCESS)
7813 break;
7814 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7815 {
7816 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7817 rc = VINF_EM_RAW_INTERRUPT;
7818 break;
7819 }
7820 }
7821
7822 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7823 return rc;
7824}
7825
7826
7827/**
7828 * Single steps guest code using VT-x.
7829 *
7830 * @returns VBox status code.
7831 * @param pVM Pointer to the VM.
7832 * @param pVCpu Pointer to the VMCPU.
7833 * @param pCtx Pointer to the guest-CPU context.
7834 *
7835 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7836 * @remarks Called with preemption disabled.
7837 */
7838static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7839{
7840 VMXTRANSIENT VmxTransient;
7841 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7842 int rc = VERR_INTERNAL_ERROR_5;
7843 uint32_t cLoops = 0;
7844 uint16_t uCsStart = pCtx->cs.Sel;
7845 uint64_t uRipStart = pCtx->rip;
7846
7847 for (;; cLoops++)
7848 {
7849 Assert(!HMR0SuspendPending());
7850 HMVMX_ASSERT_CPU_SAFE();
7851
7852 /* Preparatory work for running guest code, this may force us to return
7853 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7854 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7855 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7856 if (rc != VINF_SUCCESS)
7857 break;
7858
7859 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7860 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7861 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7862
7863 /* Restore any residual host-state and save any bits shared between host
7864 and guest into the guest-CPU state. Re-enables interrupts! */
7865 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7866
7867 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7868 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7869 {
7870 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7871 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7872 return rc;
7873 }
7874
7875 /* Handle the VM-exit. */
7876 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7877 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
7878 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7879 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7880 HMVMX_START_EXIT_DISPATCH_PROF();
7881#ifdef HMVMX_USE_FUNCTION_TABLE
7882 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7883#else
7884 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7885#endif
7886 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7887 if (rc != VINF_SUCCESS)
7888 break;
7889 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7890 {
7891 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7892 rc = VINF_EM_RAW_INTERRUPT;
7893 break;
7894 }
7895
7896 /*
7897 * Did the RIP change, if so, consider it a single step.
7898 * Otherwise, make sure one of the TFs gets set.
7899 */
7900 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7901 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7902 AssertRCReturn(rc2, rc2);
7903 if ( pCtx->rip != uRipStart
7904 || pCtx->cs.Sel != uCsStart)
7905 {
7906 rc = VINF_EM_DBG_STEPPED;
7907 break;
7908 }
7909 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
7910 }
7911
7912 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7913 return rc;
7914}
7915
7916
7917/**
7918 * Runs the guest code using VT-x.
7919 *
7920 * @returns VBox status code.
7921 * @param pVM Pointer to the VM.
7922 * @param pVCpu Pointer to the VMCPU.
7923 * @param pCtx Pointer to the guest-CPU context.
7924 *
7925 * @remarks Called with preemption disabled.
7926 */
7927VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7928{
7929 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7930 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
7931 HMVMX_ASSERT_PREEMPT_SAFE();
7932
7933 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
7934
7935 int rc;
7936 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7937 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7938 else
7939 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7940
7941 if (rc == VERR_EM_INTERPRETER)
7942 rc = VINF_EM_RAW_EMULATE_INSTR;
7943 else if (rc == VINF_EM_RESET)
7944 rc = VINF_EM_TRIPLE_FAULT;
7945
7946 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7947 if (RT_FAILURE(rc2))
7948 {
7949 pVCpu->hm.s.u32HMError = rc;
7950 rc = rc2;
7951 }
7952 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
7953 return rc;
7954}
7955
7956
7957#ifndef HMVMX_USE_FUNCTION_TABLE
7958DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7959{
7960 int rc;
7961 switch (rcReason)
7962 {
7963 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7964 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7965 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7966 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7967 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7968 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7969 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7970 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7971 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7972 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7973 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7974 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7975 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7976 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7977 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7978 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7979 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7980 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7981 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7982 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7983 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7984 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7985 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7986 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7987 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7988 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7989 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7990 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7991 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7992 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7993 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7994 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7995 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7996
7997 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7998 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7999 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8000 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8001 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8002 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8003 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8004 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8005 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8006
8007 case VMX_EXIT_VMCALL:
8008 case VMX_EXIT_VMCLEAR:
8009 case VMX_EXIT_VMLAUNCH:
8010 case VMX_EXIT_VMPTRLD:
8011 case VMX_EXIT_VMPTRST:
8012 case VMX_EXIT_VMREAD:
8013 case VMX_EXIT_VMRESUME:
8014 case VMX_EXIT_VMWRITE:
8015 case VMX_EXIT_VMXOFF:
8016 case VMX_EXIT_VMXON:
8017 case VMX_EXIT_INVEPT:
8018 case VMX_EXIT_INVVPID:
8019 case VMX_EXIT_VMFUNC:
8020 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8021 break;
8022 default:
8023 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8024 break;
8025 }
8026 return rc;
8027}
8028#endif
8029
8030#ifdef DEBUG
8031/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8032# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8033 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8034
8035# define HMVMX_ASSERT_PREEMPT_CPUID() \
8036 do \
8037 { \
8038 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8039 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8040 } while (0)
8041
8042# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8043 do { \
8044 AssertPtr(pVCpu); \
8045 AssertPtr(pMixedCtx); \
8046 AssertPtr(pVmxTransient); \
8047 Assert(pVmxTransient->fVMEntryFailed == false); \
8048 Assert(ASMIntAreEnabled()); \
8049 HMVMX_ASSERT_PREEMPT_SAFE(); \
8050 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8051 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8052 HMVMX_ASSERT_PREEMPT_SAFE(); \
8053 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8054 HMVMX_ASSERT_PREEMPT_CPUID(); \
8055 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8056 } while (0)
8057
8058# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8059 do { \
8060 Log4Func(("\n")); \
8061 } while(0)
8062#else /* Release builds */
8063# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8064# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8065#endif
8066
8067
8068/**
8069 * Advances the guest RIP after reading it from the VMCS.
8070 *
8071 * @returns VBox status code.
8072 * @param pVCpu Pointer to the VMCPU.
8073 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8074 * out-of-sync. Make sure to update the required fields
8075 * before using them.
8076 * @param pVmxTransient Pointer to the VMX transient structure.
8077 *
8078 * @remarks No-long-jump zone!!!
8079 */
8080DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8081{
8082 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8083 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8084 AssertRCReturn(rc, rc);
8085
8086 pMixedCtx->rip += pVmxTransient->cbInstr;
8087 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
8088 return rc;
8089}
8090
8091
8092/**
8093 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8094 * and update error record fields accordingly.
8095 *
8096 * @return VMX_IGS_* return codes.
8097 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8098 * wrong with the guest state.
8099 *
8100 * @param pVM Pointer to the VM.
8101 * @param pVCpu Pointer to the VMCPU.
8102 * @param pCtx Pointer to the guest-CPU state.
8103 */
8104static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8105{
8106#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8107#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8108 uError = (err); \
8109 break; \
8110 } else do {} while (0)
8111/* Duplicate of IEM_IS_CANONICAL(). */
8112#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8113
8114 int rc;
8115 uint32_t uError = VMX_IGS_ERROR;
8116 uint32_t u32Val;
8117 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8118
8119 do
8120 {
8121 /*
8122 * CR0.
8123 */
8124 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8125 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8126 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8127 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8128 if (fUnrestrictedGuest)
8129 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8130
8131 uint32_t u32GuestCR0;
8132 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8133 AssertRCBreak(rc);
8134 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8135 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8136 if ( !fUnrestrictedGuest
8137 && (u32GuestCR0 & X86_CR0_PG)
8138 && !(u32GuestCR0 & X86_CR0_PE))
8139 {
8140 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8141 }
8142
8143 /*
8144 * CR4.
8145 */
8146 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8147 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8148
8149 uint32_t u32GuestCR4;
8150 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8151 AssertRCBreak(rc);
8152 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8153 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8154
8155 /*
8156 * IA32_DEBUGCTL MSR.
8157 */
8158 uint64_t u64Val;
8159 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8160 AssertRCBreak(rc);
8161 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8162 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8163 {
8164 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8165 }
8166 uint64_t u64DebugCtlMsr = u64Val;
8167
8168#ifdef VBOX_STRICT
8169 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8170 AssertRCBreak(rc);
8171 Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8172#endif
8173 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8174
8175 /*
8176 * RIP and RFLAGS.
8177 */
8178 uint32_t u32Eflags;
8179#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8180 if (HMVMX_IS_64BIT_HOST_MODE())
8181 {
8182 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8183 AssertRCBreak(rc);
8184 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8185 if ( !fLongModeGuest
8186 || !pCtx->cs.Attr.n.u1Long)
8187 {
8188 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8189 }
8190 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8191 * must be identical if the "IA32e mode guest" VM-entry control is 1
8192 * and CS.L is 1. No check applies if the CPU supports 64
8193 * linear-address bits. */
8194
8195 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8196 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8197 AssertRCBreak(rc);
8198 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8199 VMX_IGS_RFLAGS_RESERVED);
8200 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8201 u32Eflags = u64Val;
8202 }
8203 else
8204#endif
8205 {
8206 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8207 AssertRCBreak(rc);
8208 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8209 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8210 }
8211
8212 if ( fLongModeGuest
8213 || ( fUnrestrictedGuest
8214 && !(u32GuestCR0 & X86_CR0_PE)))
8215 {
8216 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8217 }
8218
8219 uint32_t u32EntryInfo;
8220 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8221 AssertRCBreak(rc);
8222 if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8223 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8224 {
8225 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8226 }
8227
8228 /*
8229 * 64-bit checks.
8230 */
8231#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8232 if (HMVMX_IS_64BIT_HOST_MODE())
8233 {
8234 if ( fLongModeGuest
8235 && !fUnrestrictedGuest)
8236 {
8237 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8238 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8239 }
8240
8241 if ( !fLongModeGuest
8242 && (u32GuestCR4 & X86_CR4_PCIDE))
8243 {
8244 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8245 }
8246
8247 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8248 * 51:32 beyond the processor's physical-address width are 0. */
8249
8250 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8251 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8252 {
8253 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8254 }
8255
8256 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8257 AssertRCBreak(rc);
8258 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8259
8260 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8261 AssertRCBreak(rc);
8262 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8263 }
8264#endif
8265
8266 /*
8267 * PERF_GLOBAL MSR.
8268 */
8269 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8270 {
8271 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8272 AssertRCBreak(rc);
8273 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8274 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8275 }
8276
8277 /*
8278 * PAT MSR.
8279 */
8280 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8281 {
8282 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8283 AssertRCBreak(rc);
8284 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8285 for (unsigned i = 0; i < 8; i++)
8286 {
8287 uint8_t u8Val = (u64Val & 0x7);
8288 if ( u8Val != 0 /* UC */
8289 || u8Val != 1 /* WC */
8290 || u8Val != 4 /* WT */
8291 || u8Val != 5 /* WP */
8292 || u8Val != 6 /* WB */
8293 || u8Val != 7 /* UC- */)
8294 {
8295 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8296 }
8297 u64Val >>= 3;
8298 }
8299 }
8300
8301 /*
8302 * EFER MSR.
8303 */
8304 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8305 {
8306 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8307 AssertRCBreak(rc);
8308 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8309 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8310 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8311 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8312 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8313 || (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8314 }
8315
8316 /*
8317 * Segment registers.
8318 */
8319 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8320 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8321 if (!(u32Eflags & X86_EFL_VM))
8322 {
8323 /* CS */
8324 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8325 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8326 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8327 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8328 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8329 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8330 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8331 /* CS cannot be loaded with NULL in protected mode. */
8332 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8333 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8334 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8335 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8336 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8337 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8338 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8339 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8340 else
8341 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8342
8343 /* SS */
8344 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8345 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8346 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8347 if ( !(pCtx->cr0 & X86_CR0_PE)
8348 || pCtx->cs.Attr.n.u4Type == 3)
8349 {
8350 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8351 }
8352 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8353 {
8354 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8355 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8356 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8357 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8358 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8359 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8360 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8361 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8362 }
8363
8364 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8365 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8366 {
8367 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8368 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8369 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8370 || pCtx->ds.Attr.n.u4Type > 11
8371 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8372 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8373 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8374 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8375 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8376 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8377 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8378 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8379 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8380 }
8381 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8382 {
8383 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8384 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8385 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8386 || pCtx->es.Attr.n.u4Type > 11
8387 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8388 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8389 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8390 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8391 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8392 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8393 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8394 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8395 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8396 }
8397 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8398 {
8399 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8400 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8401 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8402 || pCtx->fs.Attr.n.u4Type > 11
8403 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8404 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8405 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8406 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8407 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8408 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8409 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8410 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8411 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8412 }
8413 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8414 {
8415 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8416 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8417 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8418 || pCtx->gs.Attr.n.u4Type > 11
8419 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8420 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8421 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8422 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8423 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8424 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8425 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8426 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8427 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8428 }
8429 /* 64-bit capable CPUs. */
8430#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8431 if (HMVMX_IS_64BIT_HOST_MODE())
8432 {
8433 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8434 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8435 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8436 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8437 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8438 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8439 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8440 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8441 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8442 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8443 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8444 }
8445#endif
8446 }
8447 else
8448 {
8449 /* V86 mode checks. */
8450 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8451 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8452 {
8453 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8454 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8455 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8456 }
8457 else
8458 {
8459 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8460 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8461 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8462 }
8463
8464 /* CS */
8465 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8466 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8467 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8468 /* SS */
8469 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8470 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8471 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8472 /* DS */
8473 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8474 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8475 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8476 /* ES */
8477 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8478 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8479 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8480 /* FS */
8481 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8482 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8483 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8484 /* GS */
8485 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8486 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8487 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8488 /* 64-bit capable CPUs. */
8489#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8490 if (HMVMX_IS_64BIT_HOST_MODE())
8491 {
8492 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8493 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8494 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8495 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8496 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8497 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8498 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8499 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8500 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8501 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8502 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8503 }
8504#endif
8505 }
8506
8507 /*
8508 * TR.
8509 */
8510 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8511 /* 64-bit capable CPUs. */
8512#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8513 if (HMVMX_IS_64BIT_HOST_MODE())
8514 {
8515 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8516 }
8517#endif
8518 if (fLongModeGuest)
8519 {
8520 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8521 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8522 }
8523 else
8524 {
8525 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8526 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8527 VMX_IGS_TR_ATTR_TYPE_INVALID);
8528 }
8529 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8530 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8531 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8532 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8533 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8534 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8535 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8536 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8537
8538 /*
8539 * GDTR and IDTR.
8540 */
8541#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8542 if (HMVMX_IS_64BIT_HOST_MODE())
8543 {
8544 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8545 AssertRCBreak(rc);
8546 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8547
8548 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8549 AssertRCBreak(rc);
8550 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8551 }
8552#endif
8553
8554 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8555 AssertRCBreak(rc);
8556 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8557
8558 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8559 AssertRCBreak(rc);
8560 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8561
8562 /*
8563 * Guest Non-Register State.
8564 */
8565 /* Activity State. */
8566 uint32_t u32ActivityState;
8567 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8568 AssertRCBreak(rc);
8569 HMVMX_CHECK_BREAK( !u32ActivityState
8570 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8571 VMX_IGS_ACTIVITY_STATE_INVALID);
8572 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8573 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8574 uint32_t u32IntrState;
8575 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8576 AssertRCBreak(rc);
8577 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8578 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8579 {
8580 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8581 }
8582
8583 /** @todo Activity state and injecting interrupts. Left as a todo since we
8584 * currently don't use activity states but ACTIVE. */
8585
8586 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8587 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8588
8589 /* Guest interruptibility-state. */
8590 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8591 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8592 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8593 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8594 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8595 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8596 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8597 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8598 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8599 if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8600 {
8601 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8602 {
8603 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8604 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8605 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8606 }
8607 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8608 {
8609 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8610 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8611 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8612 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8613 }
8614 }
8615 /** @todo Assumes the processor is not in SMM. */
8616 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8617 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8618 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8619 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8620 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8621 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8622 && VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8623 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8624 {
8625 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8626 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8627 }
8628
8629 /* Pending debug exceptions. */
8630 if (HMVMX_IS_64BIT_HOST_MODE())
8631 {
8632 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8633 AssertRCBreak(rc);
8634 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8635 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8636 u32Val = u64Val; /* For pending debug exceptions checks below. */
8637 }
8638 else
8639 {
8640 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8641 AssertRCBreak(rc);
8642 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8643 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8644 }
8645
8646 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8647 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8648 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8649 {
8650 if ( (u32Eflags & X86_EFL_TF)
8651 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8652 {
8653 /* Bit 14 is PendingDebug.BS. */
8654 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8655 }
8656 if ( !(u32Eflags & X86_EFL_TF)
8657 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8658 {
8659 /* Bit 14 is PendingDebug.BS. */
8660 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8661 }
8662 }
8663
8664 /* VMCS link pointer. */
8665 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8666 AssertRCBreak(rc);
8667 if (u64Val != UINT64_C(0xffffffffffffffff))
8668 {
8669 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8670 /** @todo Bits beyond the processor's physical-address width MBZ. */
8671 /** @todo 32-bit located in memory referenced by value of this field (as a
8672 * physical address) must contain the processor's VMCS revision ID. */
8673 /** @todo SMM checks. */
8674 }
8675
8676 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8677
8678 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8679 if (uError == VMX_IGS_ERROR)
8680 uError = VMX_IGS_REASON_NOT_FOUND;
8681 } while (0);
8682
8683 pVCpu->hm.s.u32HMError = uError;
8684 return uError;
8685
8686#undef HMVMX_ERROR_BREAK
8687#undef HMVMX_CHECK_BREAK
8688#undef HMVMX_IS_CANONICAL
8689}
8690
8691/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8692/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8693/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8694
8695/** @name VM-exit handlers.
8696 * @{
8697 */
8698
8699/**
8700 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8701 */
8702HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8703{
8704 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8705 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8706 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8707#if HC_ARCH_BITS == 64
8708 Assert(ASMIntAreEnabled());
8709 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8710 return VINF_SUCCESS;
8711#endif
8712 return VINF_EM_RAW_INTERRUPT;
8713}
8714
8715
8716/**
8717 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8718 */
8719HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8720{
8721 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8722 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8723
8724 int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8725 AssertRCReturn(rc, rc);
8726
8727 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8728 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8729 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8730 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8731
8732 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8733 {
8734 /*
8735 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8736 * anything we inject is not going to cause a VM-exit directly for the event being injected.
8737 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8738 *
8739 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8740 */
8741 VMXDispatchHostNmi();
8742 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
8743 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8744 return VINF_SUCCESS;
8745 }
8746
8747 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8748 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8749 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8750 {
8751 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8752 return VINF_SUCCESS;
8753 }
8754 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8755 {
8756 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8757 return rc;
8758 }
8759
8760 uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8761 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8762 switch (uIntrType)
8763 {
8764 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8765 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8766 /* no break */
8767 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8768 {
8769 switch (uVector)
8770 {
8771 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8772 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8773 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8774 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8775 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8776 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8777#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8778 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8779 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8780 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8781 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8782 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8783 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8784 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8785 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8786 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8787 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8788#endif
8789 default:
8790 {
8791 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8792 AssertRCReturn(rc, rc);
8793
8794 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8795 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8796 {
8797 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8798 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8799 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8800
8801 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8802 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8803 AssertRCReturn(rc, rc);
8804 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8805 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8806 0 /* GCPtrFaultAddress */);
8807 AssertRCReturn(rc, rc);
8808 }
8809 else
8810 {
8811 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8812 pVCpu->hm.s.u32HMError = uVector;
8813 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8814 }
8815 break;
8816 }
8817 }
8818 break;
8819 }
8820
8821 default:
8822 {
8823 pVCpu->hm.s.u32HMError = uExitIntrInfo;
8824 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8825 AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8826 break;
8827 }
8828 }
8829 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8830 return rc;
8831}
8832
8833
8834/**
8835 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8836 */
8837HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8838{
8839 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8840
8841 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8842 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8843 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8844 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8845 AssertRCReturn(rc, rc);
8846
8847 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8848 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8849 return VINF_SUCCESS;
8850}
8851
8852
8853/**
8854 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8855 */
8856HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8857{
8858 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8859 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8860 pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8861 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8862}
8863
8864
8865/**
8866 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8867 */
8868HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8869{
8870 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8871 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8872 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8873}
8874
8875
8876/**
8877 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8878 */
8879HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8880{
8881 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8882 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8883 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8884}
8885
8886
8887/**
8888 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8889 */
8890HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8891{
8892 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8893 PVM pVM = pVCpu->CTX_SUFF(pVM);
8894 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8895 if (RT_LIKELY(rc == VINF_SUCCESS))
8896 {
8897 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8898 Assert(pVmxTransient->cbInstr == 2);
8899 }
8900 else
8901 {
8902 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8903 rc = VERR_EM_INTERPRETER;
8904 }
8905 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8906 return rc;
8907}
8908
8909
8910/**
8911 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8912 */
8913HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8914{
8915 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8916 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8917 AssertRCReturn(rc, rc);
8918
8919 if (pMixedCtx->cr4 & X86_CR4_SMXE)
8920 return VINF_EM_RAW_EMULATE_INSTR;
8921
8922 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8923 pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8924 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8925}
8926
8927
8928/**
8929 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8930 */
8931HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8932{
8933 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8934 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8935 AssertRCReturn(rc, rc);
8936
8937 PVM pVM = pVCpu->CTX_SUFF(pVM);
8938 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8939 if (RT_LIKELY(rc == VINF_SUCCESS))
8940 {
8941 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8942 Assert(pVmxTransient->cbInstr == 2);
8943 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8944 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8945 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8946 }
8947 else
8948 {
8949 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8950 rc = VERR_EM_INTERPRETER;
8951 }
8952 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8953 return rc;
8954}
8955
8956
8957/**
8958 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8959 */
8960HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8961{
8962 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8963 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8964 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8965 AssertRCReturn(rc, rc);
8966
8967 PVM pVM = pVCpu->CTX_SUFF(pVM);
8968 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8969 if (RT_LIKELY(rc == VINF_SUCCESS))
8970 {
8971 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8972 Assert(pVmxTransient->cbInstr == 3);
8973 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8974 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8975 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8976 }
8977 else
8978 {
8979 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8980 rc = VERR_EM_INTERPRETER;
8981 }
8982 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8983 return rc;
8984}
8985
8986
8987/**
8988 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8989 */
8990HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8991{
8992 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8993 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8994 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8995 AssertRCReturn(rc, rc);
8996
8997 PVM pVM = pVCpu->CTX_SUFF(pVM);
8998 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8999 if (RT_LIKELY(rc == VINF_SUCCESS))
9000 {
9001 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9002 Assert(pVmxTransient->cbInstr == 2);
9003 }
9004 else
9005 {
9006 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9007 rc = VERR_EM_INTERPRETER;
9008 }
9009 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9010 return rc;
9011}
9012
9013
9014/**
9015 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9016 */
9017HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9018{
9019 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9020 PVM pVM = pVCpu->CTX_SUFF(pVM);
9021 Assert(!pVM->hm.s.fNestedPaging);
9022
9023 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9024 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9025 AssertRCReturn(rc, rc);
9026
9027 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9028 rc = VBOXSTRICTRC_VAL(rc2);
9029 if (RT_LIKELY(rc == VINF_SUCCESS))
9030 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9031 else
9032 {
9033 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9034 pVmxTransient->uExitQualification, rc));
9035 }
9036 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9037 return rc;
9038}
9039
9040
9041/**
9042 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9043 */
9044HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9045{
9046 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9047 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9048 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9049 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9050 AssertRCReturn(rc, rc);
9051
9052 PVM pVM = pVCpu->CTX_SUFF(pVM);
9053 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9054 if (RT_LIKELY(rc == VINF_SUCCESS))
9055 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9056 else
9057 {
9058 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9059 rc = VERR_EM_INTERPRETER;
9060 }
9061 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9062 return rc;
9063}
9064
9065
9066/**
9067 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9068 */
9069HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9070{
9071 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9072 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9073 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9074 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9075 AssertRCReturn(rc, rc);
9076
9077 PVM pVM = pVCpu->CTX_SUFF(pVM);
9078 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9079 rc = VBOXSTRICTRC_VAL(rc2);
9080 if (RT_LIKELY( rc == VINF_SUCCESS
9081 || rc == VINF_EM_HALT))
9082 {
9083 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9084 AssertRCReturn(rc3, rc3);
9085
9086 if ( rc == VINF_EM_HALT
9087 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9088 {
9089 rc = VINF_SUCCESS;
9090 }
9091 }
9092 else
9093 {
9094 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9095 rc = VERR_EM_INTERPRETER;
9096 }
9097 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
9098 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9099 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9100 return rc;
9101}
9102
9103
9104/**
9105 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9106 */
9107HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9108{
9109 /*
9110 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9111 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9112 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
9113 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9114 */
9115 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9116 pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
9117 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9118}
9119
9120
9121/**
9122 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9123 */
9124HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9125{
9126 /*
9127 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9128 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9129 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9130 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9131 */
9132 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9133 pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
9134 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9135}
9136
9137
9138/**
9139 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9140 */
9141HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9142{
9143 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9144 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9145 pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
9146 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9147}
9148
9149
9150/**
9151 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9152 */
9153HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9154{
9155 /*
9156 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9157 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9158 * See Intel spec. 25.3 "Other Causes of VM-exits".
9159 */
9160 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9161 pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
9162 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9163}
9164
9165
9166/**
9167 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9168 * VM-exit.
9169 */
9170HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9171{
9172 /*
9173 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9174 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9175 *
9176 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9177 * See Intel spec. "23.8 Restrictions on VMX operation".
9178 */
9179 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9180 return VINF_SUCCESS;
9181}
9182
9183
9184/**
9185 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9186 * VM-exit.
9187 */
9188HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9189{
9190 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9191 return VINF_EM_RESET;
9192}
9193
9194
9195/**
9196 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9197 */
9198HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9199{
9200 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9201 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9202 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9203 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9204 AssertRCReturn(rc, rc);
9205
9206 pMixedCtx->rip++;
9207 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9208 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9209 rc = VINF_SUCCESS;
9210 else
9211 rc = VINF_EM_HALT;
9212
9213 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9214 return rc;
9215}
9216
9217
9218/**
9219 * VM-exit handler for instructions that result in a #UD exception delivered to
9220 * the guest.
9221 */
9222HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9223{
9224 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9225 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9226 return VINF_SUCCESS;
9227}
9228
9229
9230/**
9231 * VM-exit handler for expiry of the VMX preemption timer.
9232 */
9233HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9234{
9235 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9236
9237 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9238 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9239
9240 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9241 PVM pVM = pVCpu->CTX_SUFF(pVM);
9242 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9243 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9244 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9245}
9246
9247
9248/**
9249 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9250 */
9251HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9252{
9253 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9254
9255 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9256 /** @todo check if XSETBV is supported by the recompiler. */
9257 return VERR_EM_INTERPRETER;
9258}
9259
9260
9261/**
9262 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9263 */
9264HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9265{
9266 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9267
9268 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9269 /** @todo implement EMInterpretInvpcid() */
9270 return VERR_EM_INTERPRETER;
9271}
9272
9273
9274/**
9275 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9276 * Error VM-exit.
9277 */
9278HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9279{
9280 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9281 AssertRCReturn(rc, rc);
9282
9283 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9284 NOREF(uInvalidReason);
9285
9286#ifdef VBOX_STRICT
9287 uint32_t uIntrState;
9288 HMVMXHCUINTREG uHCReg;
9289 uint64_t u64Val;
9290 uint32_t u32Val;
9291
9292 rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
9293 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9294 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9295 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9296 AssertRCReturn(rc, rc);
9297
9298 Log4(("uInvalidReason %u\n", uInvalidReason));
9299 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
9300 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9301 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9302 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9303
9304 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9305 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9306 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9307 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9308 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9309 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9310 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9311 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9312 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9313 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9314 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9315 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9316#endif
9317
9318 PVM pVM = pVCpu->CTX_SUFF(pVM);
9319 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9320
9321 return VERR_VMX_INVALID_GUEST_STATE;
9322}
9323
9324
9325/**
9326 * VM-exit handler for VM-entry failure due to an MSR-load
9327 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9328 */
9329HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9330{
9331 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9332 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9333}
9334
9335
9336/**
9337 * VM-exit handler for VM-entry failure due to a machine-check event
9338 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9339 */
9340HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9341{
9342 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9343 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9344}
9345
9346
9347/**
9348 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9349 * theory.
9350 */
9351HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9352{
9353 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9354 return VERR_VMX_UNDEFINED_EXIT_CODE;
9355}
9356
9357
9358/**
9359 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9360 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9361 * Conditional VM-exit.
9362 */
9363HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9364{
9365 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9366
9367 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9368 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9369 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9370 return VERR_EM_INTERPRETER;
9371 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9372 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9373}
9374
9375
9376/**
9377 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9378 */
9379HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9380{
9381 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9382
9383 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9384 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9385 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9386 return VERR_EM_INTERPRETER;
9387 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9388 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9389}
9390
9391
9392/**
9393 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9394 */
9395HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9396{
9397 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9398
9399 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9400 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9401 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9402 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9403 AssertRCReturn(rc, rc);
9404 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9405
9406 PVM pVM = pVCpu->CTX_SUFF(pVM);
9407 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9408 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9409 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9410 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9411
9412 if (RT_LIKELY(rc == VINF_SUCCESS))
9413 {
9414 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9415 Assert(pVmxTransient->cbInstr == 2);
9416 }
9417 return rc;
9418}
9419
9420
9421/**
9422 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9423 */
9424HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9425{
9426 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9427 PVM pVM = pVCpu->CTX_SUFF(pVM);
9428 int rc = VINF_SUCCESS;
9429
9430 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9431 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9432 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9433 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9434 AssertRCReturn(rc, rc);
9435 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9436
9437 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9438 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9439 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9440
9441 if (RT_LIKELY(rc == VINF_SUCCESS))
9442 {
9443 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9444
9445 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9446 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9447 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9448 {
9449 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9450 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9451 EMInterpretWrmsr() changes it. */
9452 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9453 }
9454 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9455 {
9456 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9457 AssertRCReturn(rc, rc);
9458 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
9459 }
9460 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9461 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9462
9463 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9464 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9465 {
9466 switch (pMixedCtx->ecx)
9467 {
9468 case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
9469 case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
9470 case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
9471 case MSR_K8_FS_BASE: /* no break */
9472 case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS; break;
9473 case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
9474 }
9475 }
9476#ifdef VBOX_STRICT
9477 else
9478 {
9479 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9480 switch (pMixedCtx->ecx)
9481 {
9482 case MSR_IA32_SYSENTER_CS:
9483 case MSR_IA32_SYSENTER_EIP:
9484 case MSR_IA32_SYSENTER_ESP:
9485 case MSR_K8_FS_BASE:
9486 case MSR_K8_GS_BASE:
9487 {
9488 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9489 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9490 }
9491
9492 case MSR_K8_LSTAR:
9493 case MSR_K6_STAR:
9494 case MSR_K8_SF_MASK:
9495 case MSR_K8_TSC_AUX:
9496 case MSR_K8_KERNEL_GS_BASE:
9497 {
9498 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9499 pMixedCtx->ecx));
9500 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9501 }
9502 }
9503 }
9504#endif /* VBOX_STRICT */
9505 }
9506 return rc;
9507}
9508
9509
9510/**
9511 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9512 */
9513HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9514{
9515 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9516
9517 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9518 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9519 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9520 return VERR_EM_INTERPRETER;
9521 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9522 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9523}
9524
9525
9526/**
9527 * VM-exit handler for when the TPR value is lowered below the specified
9528 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9529 */
9530HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9531{
9532 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9533 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9534
9535 /*
9536 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9537 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9538 * resume guest execution.
9539 */
9540 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9541 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9542 return VINF_SUCCESS;
9543}
9544
9545
9546/**
9547 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9548 * VM-exit.
9549 *
9550 * @retval VINF_SUCCESS when guest execution can continue.
9551 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9552 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9553 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9554 * recompiler.
9555 */
9556HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9557{
9558 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9559 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9560 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9561 AssertRCReturn(rc, rc);
9562
9563 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9564 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9565 PVM pVM = pVCpu->CTX_SUFF(pVM);
9566 switch (uAccessType)
9567 {
9568 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9569 {
9570#if 0
9571 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9572 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9573#else
9574 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9575 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9576 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9577#endif
9578 AssertRCReturn(rc, rc);
9579
9580 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9581 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9582 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9583 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9584
9585 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9586 {
9587 case 0: /* CR0 */
9588 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9589 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9590 break;
9591 case 2: /* C2 **/
9592 /* Nothing to do here, CR2 it's not part of the VMCS. */
9593 break;
9594 case 3: /* CR3 */
9595 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9596 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9597 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR3;
9598 break;
9599 case 4: /* CR4 */
9600 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9601 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR4;
9602 break;
9603 case 8: /* CR8 */
9604 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9605 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9606 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9607 break;
9608 default:
9609 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9610 break;
9611 }
9612
9613 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9614 break;
9615 }
9616
9617 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9618 {
9619 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9620 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9621 AssertRCReturn(rc, rc);
9622 Assert( !pVM->hm.s.fNestedPaging
9623 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9624 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9625
9626 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9627 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9628 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9629
9630 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9631 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9632 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9633 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9634 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9635 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9636 break;
9637 }
9638
9639 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9640 {
9641 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9642 AssertRCReturn(rc, rc);
9643 rc = EMInterpretCLTS(pVM, pVCpu);
9644 AssertRCReturn(rc, rc);
9645 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9646 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9647 Log4(("CRX CLTS write rc=%d\n", rc));
9648 break;
9649 }
9650
9651 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9652 {
9653 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9654 AssertRCReturn(rc, rc);
9655 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9656 if (RT_LIKELY(rc == VINF_SUCCESS))
9657 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9658 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9659 Log4(("CRX LMSW write rc=%d\n", rc));
9660 break;
9661 }
9662
9663 default:
9664 {
9665 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9666 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9667 }
9668 }
9669
9670 /* Validate possible error codes. */
9671 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9672 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9673 if (RT_SUCCESS(rc))
9674 {
9675 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9676 AssertRCReturn(rc2, rc2);
9677 }
9678
9679 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9680 return rc;
9681}
9682
9683
9684/**
9685 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9686 * VM-exit.
9687 */
9688HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9689{
9690 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9691 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9692
9693 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9694 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9695 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9696 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9697 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9698 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9699 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9700 AssertRCReturn(rc2, rc2);
9701
9702 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9703 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9704 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9705 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9706 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9707 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9708 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9709
9710 /* I/O operation lookup arrays. */
9711 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9712 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9713
9714 VBOXSTRICTRC rcStrict;
9715 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9716 const uint32_t cbInstr = pVmxTransient->cbInstr;
9717 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9718 PVM pVM = pVCpu->CTX_SUFF(pVM);
9719 if (fIOString)
9720 {
9721 /*
9722 * INS/OUTS - I/O String instruction.
9723 *
9724 * Use instruction-information if available, otherwise fall back on
9725 * interpreting the instruction.
9726 */
9727 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9728#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9729 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9730 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9731 {
9732 rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9733 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9734 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9735 AssertRCReturn(rc2, rc2);
9736 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9737 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9738 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9739 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9740 if (fIOWrite)
9741 {
9742 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9743 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9744 //if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9745 // hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9746 // pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9747 }
9748 else
9749 {
9750 AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9751 ("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9752 VERR_HMVMX_IPE_4);
9753 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9754 //if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9755 // hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9756 }
9757 }
9758 else
9759 {
9760 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9761 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9762 AssertRCReturn(rc2, rc2);
9763 rcStrict = IEMExecOne(pVCpu);
9764 }
9765 /** @todo IEM needs to be setting these flags somehow. */
9766 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9767 fUpdateRipAlready = true;
9768#else
9769 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9770 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9771 if (RT_SUCCESS(rcStrict))
9772 {
9773 if (fIOWrite)
9774 {
9775 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9776 (DISCPUMODE)pDis->uAddrMode, cbValue);
9777 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9778 }
9779 else
9780 {
9781 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9782 (DISCPUMODE)pDis->uAddrMode, cbValue);
9783 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9784 }
9785 }
9786 else
9787 {
9788 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9789 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9790 }
9791#endif
9792 }
9793 else
9794 {
9795 /*
9796 * IN/OUT - I/O instruction.
9797 */
9798 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9799 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9800 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9801 if (fIOWrite)
9802 {
9803 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9804 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9805 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9806 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9807 }
9808 else
9809 {
9810 uint32_t u32Result = 0;
9811 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9812 if (IOM_SUCCESS(rcStrict))
9813 {
9814 /* Save result of I/O IN instr. in AL/AX/EAX. */
9815 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
9816 }
9817 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9818 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9819 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9820 }
9821 }
9822
9823 if (IOM_SUCCESS(rcStrict))
9824 {
9825 if (!fUpdateRipAlready)
9826 {
9827 pMixedCtx->rip += cbInstr;
9828 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9829 }
9830
9831 /*
9832 * If any I/O breakpoints are armed, we need to check if one triggered
9833 * and take appropriate action.
9834 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9835 */
9836 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9837 AssertRCReturn(rc2, rc2);
9838
9839 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9840 * execution engines about whether hyper BPs and such are pending. */
9841 uint32_t const uDr7 = pMixedCtx->dr[7];
9842 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9843 && X86_DR7_ANY_RW_IO(uDr7)
9844 && (pMixedCtx->cr4 & X86_CR4_DE))
9845 || DBGFBpIsHwIoArmed(pVM)))
9846 {
9847 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9848
9849 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
9850 VMMRZCallRing3Disable(pVCpu);
9851 HM_DISABLE_PREEMPT_IF_NEEDED();
9852
9853 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
9854
9855 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9856 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9857 {
9858 /* Raise #DB. */
9859 if (fIsGuestDbgActive)
9860 ASMSetDR6(pMixedCtx->dr[6]);
9861 if (pMixedCtx->dr[7] != uDr7)
9862 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9863
9864 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9865 }
9866 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9867 else if ( rcStrict2 != VINF_SUCCESS
9868 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
9869 rcStrict = rcStrict2;
9870
9871 HM_RESTORE_PREEMPT_IF_NEEDED();
9872 VMMRZCallRing3Enable(pVCpu);
9873 }
9874 }
9875
9876#ifdef DEBUG
9877 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9878 Assert(!fIOWrite);
9879 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9880 Assert(fIOWrite);
9881 else
9882 {
9883 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9884 * statuses, that the VMM device and some others may return. See
9885 * IOM_SUCCESS() for guidance. */
9886 AssertMsg( RT_FAILURE(rcStrict)
9887 || rcStrict == VINF_SUCCESS
9888 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
9889 || rcStrict == VINF_EM_DBG_BREAKPOINT
9890 || rcStrict == VINF_EM_RAW_GUEST_TRAP
9891 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9892 }
9893#endif
9894
9895 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9896 return VBOXSTRICTRC_TODO(rcStrict);
9897}
9898
9899
9900/**
9901 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9902 * VM-exit.
9903 */
9904HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9905{
9906 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9907
9908 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
9909 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9910 AssertRCReturn(rc, rc);
9911 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9912 {
9913 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9914 AssertRCReturn(rc, rc);
9915 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9916 {
9917 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9918
9919 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9920 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9921 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9922 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9923 {
9924 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9925 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9926
9927 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9928 Assert(!pVCpu->hm.s.Event.fPending);
9929 pVCpu->hm.s.Event.fPending = true;
9930 pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9931 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9932 AssertRCReturn(rc, rc);
9933 if (fErrorCodeValid)
9934 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9935 else
9936 pVCpu->hm.s.Event.u32ErrCode = 0;
9937 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9938 && uVector == X86_XCPT_PF)
9939 {
9940 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9941 }
9942
9943 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9944 }
9945 }
9946 }
9947
9948 /** @todo Emulate task switch someday, currently just going back to ring-3 for
9949 * emulation. */
9950 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9951 return VERR_EM_INTERPRETER;
9952}
9953
9954
9955/**
9956 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9957 */
9958HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9959{
9960 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9961 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9962 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9963 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9964 AssertRCReturn(rc, rc);
9965 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9966 return VINF_EM_DBG_STEPPED;
9967}
9968
9969
9970/**
9971 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9972 */
9973HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9974{
9975 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9976
9977 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9978 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9979 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9980 return VINF_SUCCESS;
9981 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9982 return rc;
9983
9984#if 0
9985 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9986 * just sync the whole thing. */
9987 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9988#else
9989 /* Aggressive state sync. for now. */
9990 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9991 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9992 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9993#endif
9994 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9995 AssertRCReturn(rc, rc);
9996
9997 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9998 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9999 switch (uAccessType)
10000 {
10001 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10002 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10003 {
10004 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10005 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10006 {
10007 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10008 }
10009
10010 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10011 GCPhys &= PAGE_BASE_GC_MASK;
10012 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10013 PVM pVM = pVCpu->CTX_SUFF(pVM);
10014 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10015 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10016
10017 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10018 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10019 CPUMCTX2CORE(pMixedCtx), GCPhys);
10020 rc = VBOXSTRICTRC_VAL(rc2);
10021 Log4(("ApicAccess rc=%d\n", rc));
10022 if ( rc == VINF_SUCCESS
10023 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10024 || rc == VERR_PAGE_NOT_PRESENT)
10025 {
10026 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10027 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10028 rc = VINF_SUCCESS;
10029 }
10030 break;
10031 }
10032
10033 default:
10034 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10035 rc = VINF_EM_RAW_EMULATE_INSTR;
10036 break;
10037 }
10038
10039 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10040 return rc;
10041}
10042
10043
10044/**
10045 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10046 * VM-exit.
10047 */
10048HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10049{
10050 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10051
10052 /* We should -not- get this VM-exit if the guest's debug registers were active. */
10053 if (pVmxTransient->fWasGuestDebugStateActive)
10054 {
10055 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10056 return VERR_VMX_UNEXPECTED_EXIT_CODE;
10057 }
10058
10059 int rc = VERR_INTERNAL_ERROR_5;
10060 if ( !DBGFIsStepping(pVCpu)
10061 && !pVCpu->hm.s.fSingleInstruction
10062 && !pVmxTransient->fWasHyperDebugStateActive)
10063 {
10064 /* Don't intercept MOV DRx and #DB any more. */
10065 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10066 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10067 AssertRCReturn(rc, rc);
10068
10069 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10070 {
10071#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10072 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10073 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10074 AssertRCReturn(rc, rc);
10075#endif
10076 }
10077
10078 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
10079 VMMRZCallRing3Disable(pVCpu);
10080 HM_DISABLE_PREEMPT_IF_NEEDED();
10081
10082 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10083 PVM pVM = pVCpu->CTX_SUFF(pVM);
10084 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10085 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
10086
10087 HM_RESTORE_PREEMPT_IF_NEEDED();
10088 VMMRZCallRing3Enable(pVCpu);
10089
10090#ifdef VBOX_WITH_STATISTICS
10091 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10092 AssertRCReturn(rc, rc);
10093 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10094 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10095 else
10096 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10097#endif
10098 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10099 return VINF_SUCCESS;
10100 }
10101
10102 /*
10103 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10104 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10105 */
10106 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10107 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10108 AssertRCReturn(rc, rc);
10109 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10110
10111 PVM pVM = pVCpu->CTX_SUFF(pVM);
10112 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10113 {
10114 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10115 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10116 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10117 if (RT_SUCCESS(rc))
10118 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
10119 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10120 }
10121 else
10122 {
10123 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10124 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10125 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10126 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10127 }
10128
10129 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10130 if (RT_SUCCESS(rc))
10131 {
10132 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10133 AssertRCReturn(rc2, rc2);
10134 }
10135 return rc;
10136}
10137
10138
10139/**
10140 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10141 * Conditional VM-exit.
10142 */
10143HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10144{
10145 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10146 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10147
10148 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10149 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10150 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10151 return VINF_SUCCESS;
10152 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10153 return rc;
10154
10155 RTGCPHYS GCPhys = 0;
10156 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10157
10158#if 0
10159 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10160#else
10161 /* Aggressive state sync. for now. */
10162 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10163 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10164 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10165#endif
10166 AssertRCReturn(rc, rc);
10167
10168 /*
10169 * If we succeed, resume guest execution.
10170 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10171 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10172 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10173 * weird case. See @bugref{6043}.
10174 */
10175 PVM pVM = pVCpu->CTX_SUFF(pVM);
10176 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10177 rc = VBOXSTRICTRC_VAL(rc2);
10178 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10179 if ( rc == VINF_SUCCESS
10180 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10181 || rc == VERR_PAGE_NOT_PRESENT)
10182 {
10183 /* Successfully handled MMIO operation. */
10184 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10185 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10186 rc = VINF_SUCCESS;
10187 }
10188 return rc;
10189}
10190
10191
10192/**
10193 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10194 * VM-exit.
10195 */
10196HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10197{
10198 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10199 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10200
10201 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10202 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10203 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10204 return VINF_SUCCESS;
10205 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10206 return rc;
10207
10208 RTGCPHYS GCPhys = 0;
10209 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10210 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10211#if 0
10212 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10213#else
10214 /* Aggressive state sync. for now. */
10215 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10216 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10217 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10218#endif
10219 AssertRCReturn(rc, rc);
10220
10221 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10222 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10223
10224 RTGCUINT uErrorCode = 0;
10225 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10226 uErrorCode |= X86_TRAP_PF_ID;
10227 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10228 uErrorCode |= X86_TRAP_PF_RW;
10229 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10230 uErrorCode |= X86_TRAP_PF_P;
10231
10232 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10233
10234 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10235 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10236
10237 /* Handle the pagefault trap for the nested shadow table. */
10238 PVM pVM = pVCpu->CTX_SUFF(pVM);
10239 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10240 TRPMResetTrap(pVCpu);
10241
10242 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10243 if ( rc == VINF_SUCCESS
10244 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10245 || rc == VERR_PAGE_NOT_PRESENT)
10246 {
10247 /* Successfully synced our nested page tables. */
10248 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10249 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10250 return VINF_SUCCESS;
10251 }
10252
10253 Log4(("EPT return to ring-3 rc=%d\n"));
10254 return rc;
10255}
10256
10257/** @} */
10258
10259/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10260/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10261/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10262
10263/** @name VM-exit exception handlers.
10264 * @{
10265 */
10266
10267/**
10268 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10269 */
10270static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10271{
10272 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10273 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10274
10275 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10276 AssertRCReturn(rc, rc);
10277
10278 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10279 {
10280 /* Old-style FPU error reporting needs some extra work. */
10281 /** @todo don't fall back to the recompiler, but do it manually. */
10282 return VERR_EM_INTERPRETER;
10283 }
10284
10285 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10286 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10287 return rc;
10288}
10289
10290
10291/**
10292 * VM-exit exception handler for #BP (Breakpoint exception).
10293 */
10294static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10295{
10296 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10297 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10298
10299 /** @todo Try optimize this by not saving the entire guest state unless
10300 * really needed. */
10301 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10302 AssertRCReturn(rc, rc);
10303
10304 PVM pVM = pVCpu->CTX_SUFF(pVM);
10305 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10306 if (rc == VINF_EM_RAW_GUEST_TRAP)
10307 {
10308 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10309 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10310 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10311 AssertRCReturn(rc, rc);
10312
10313 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10314 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10315 }
10316
10317 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10318 return rc;
10319}
10320
10321
10322/**
10323 * VM-exit exception handler for #DB (Debug exception).
10324 */
10325static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10326{
10327 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10328 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10329 Log6(("XcptDB\n"));
10330
10331 /*
10332 * Get the DR6-like values from the exit qualification and pass it to DBGF
10333 * for processing.
10334 */
10335 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10336 AssertRCReturn(rc, rc);
10337
10338 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10339 uint64_t uDR6 = X86_DR6_INIT_VAL;
10340 uDR6 |= ( pVmxTransient->uExitQualification
10341 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10342
10343 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10344 if (rc == VINF_EM_RAW_GUEST_TRAP)
10345 {
10346 /*
10347 * The exception was for the guest. Update DR6, DR7.GD and
10348 * IA32_DEBUGCTL.LBR before forwarding it.
10349 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10350 */
10351 VMMRZCallRing3Disable(pVCpu);
10352 HM_DISABLE_PREEMPT_IF_NEEDED();
10353
10354 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10355 pMixedCtx->dr[6] |= uDR6;
10356 if (CPUMIsGuestDebugStateActive(pVCpu))
10357 ASMSetDR6(pMixedCtx->dr[6]);
10358
10359 HM_RESTORE_PREEMPT_IF_NEEDED();
10360 VMMRZCallRing3Enable(pVCpu);
10361
10362 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10363 AssertRCReturn(rc, rc);
10364
10365 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10366 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10367
10368 /* Paranoia. */
10369 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10370 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10371
10372 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10373 AssertRCReturn(rc, rc);
10374
10375 /*
10376 * Raise #DB in the guest.
10377 */
10378 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10379 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10380 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10381 AssertRCReturn(rc, rc);
10382 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10383 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10384 return VINF_SUCCESS;
10385 }
10386
10387 /*
10388 * Not a guest trap, must be a hypervisor related debug event then.
10389 * Update DR6 in case someone is interested in it.
10390 */
10391 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10392 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
10393 CPUMSetHyperDR6(pVCpu, uDR6);
10394
10395 return rc;
10396}
10397
10398
10399/**
10400 * VM-exit exception handler for #NM (Device-not-available exception: floating
10401 * point exception).
10402 */
10403static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10404{
10405 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10406
10407 /* We require CR0 and EFER. EFER is always up-to-date. */
10408 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10409 AssertRCReturn(rc, rc);
10410
10411 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
10412 VMMRZCallRing3Disable(pVCpu);
10413 HM_DISABLE_PREEMPT_IF_NEEDED();
10414
10415 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
10416 if (pVmxTransient->fWasGuestFPUStateActive)
10417 {
10418 rc = VINF_EM_RAW_GUEST_TRAP;
10419 Assert(CPUMIsGuestFPUStateActive(pVCpu) || (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
10420 }
10421 else
10422 {
10423#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10424 Assert(!pVmxTransient->fWasGuestFPUStateActive);
10425#endif
10426 /* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10427 rc = CPUMR0LoadGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10428 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
10429 }
10430
10431 HM_RESTORE_PREEMPT_IF_NEEDED();
10432 VMMRZCallRing3Enable(pVCpu);
10433
10434 if (rc == VINF_SUCCESS)
10435 {
10436 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
10437 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10438 }
10439 else
10440 {
10441 /* Forward #NM to the guest. */
10442 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10443 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10444 AssertRCReturn(rc, rc);
10445 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10446 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10447 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10448 }
10449
10450 return VINF_SUCCESS;
10451}
10452
10453
10454/**
10455 * VM-exit exception handler for #GP (General-protection exception).
10456 *
10457 * @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
10458 */
10459static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10460{
10461 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10462 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10463
10464 int rc = VERR_INTERNAL_ERROR_5;
10465 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10466 {
10467#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10468 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10469 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10470 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10471 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10472 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10473 AssertRCReturn(rc, rc);
10474 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10475 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10476 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10477 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10478 return rc;
10479#else
10480 /* We don't intercept #GP. */
10481 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10482 return VERR_VMX_UNEXPECTED_EXCEPTION;
10483#endif
10484 }
10485
10486 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10487 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10488
10489 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10490 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10491 AssertRCReturn(rc, rc);
10492
10493 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10494 uint32_t cbOp = 0;
10495 PVM pVM = pVCpu->CTX_SUFF(pVM);
10496 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10497 if (RT_SUCCESS(rc))
10498 {
10499 rc = VINF_SUCCESS;
10500 Assert(cbOp == pDis->cbInstr);
10501 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10502 switch (pDis->pCurInstr->uOpcode)
10503 {
10504 case OP_CLI:
10505 {
10506 pMixedCtx->eflags.Bits.u1IF = 0;
10507 pMixedCtx->rip += pDis->cbInstr;
10508 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10509 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10510 break;
10511 }
10512
10513 case OP_STI:
10514 {
10515 pMixedCtx->eflags.Bits.u1IF = 1;
10516 pMixedCtx->rip += pDis->cbInstr;
10517 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10518 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10519 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10520 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10521 break;
10522 }
10523
10524 case OP_HLT:
10525 {
10526 rc = VINF_EM_HALT;
10527 pMixedCtx->rip += pDis->cbInstr;
10528 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
10529 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10530 break;
10531 }
10532
10533 case OP_POPF:
10534 {
10535 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10536 uint32_t cbParm = 0;
10537 uint32_t uMask = 0;
10538 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10539 {
10540 cbParm = 4;
10541 uMask = 0xffffffff;
10542 }
10543 else
10544 {
10545 cbParm = 2;
10546 uMask = 0xffff;
10547 }
10548
10549 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10550 RTGCPTR GCPtrStack = 0;
10551 X86EFLAGS Eflags;
10552 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10553 &GCPtrStack);
10554 if (RT_SUCCESS(rc))
10555 {
10556 Assert(sizeof(Eflags.u32) >= cbParm);
10557 Eflags.u32 = 0;
10558 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10559 }
10560 if (RT_FAILURE(rc))
10561 {
10562 rc = VERR_EM_INTERPRETER;
10563 break;
10564 }
10565 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10566 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10567 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10568 /* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10569 pMixedCtx->eflags.Bits.u1RF = 0;
10570 pMixedCtx->esp += cbParm;
10571 pMixedCtx->esp &= uMask;
10572 pMixedCtx->rip += pDis->cbInstr;
10573 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10574 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10575 break;
10576 }
10577
10578 case OP_PUSHF:
10579 {
10580 uint32_t cbParm = 0;
10581 uint32_t uMask = 0;
10582 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10583 {
10584 cbParm = 4;
10585 uMask = 0xffffffff;
10586 }
10587 else
10588 {
10589 cbParm = 2;
10590 uMask = 0xffff;
10591 }
10592
10593 /* Get the stack pointer & push the contents of eflags onto the stack. */
10594 RTGCPTR GCPtrStack = 0;
10595 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10596 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10597 if (RT_FAILURE(rc))
10598 {
10599 rc = VERR_EM_INTERPRETER;
10600 break;
10601 }
10602 X86EFLAGS Eflags = pMixedCtx->eflags;
10603 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10604 Eflags.Bits.u1RF = 0;
10605 Eflags.Bits.u1VM = 0;
10606
10607 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10608 if (RT_FAILURE(rc))
10609 {
10610 rc = VERR_EM_INTERPRETER;
10611 break;
10612 }
10613 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10614 pMixedCtx->esp -= cbParm;
10615 pMixedCtx->esp &= uMask;
10616 pMixedCtx->rip += pDis->cbInstr;
10617 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP;
10618 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10619 break;
10620 }
10621
10622 case OP_IRET:
10623 {
10624 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10625 * instruction reference. */
10626 RTGCPTR GCPtrStack = 0;
10627 uint32_t uMask = 0xffff;
10628 uint16_t aIretFrame[3];
10629 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10630 {
10631 rc = VERR_EM_INTERPRETER;
10632 break;
10633 }
10634 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10635 &GCPtrStack);
10636 if (RT_SUCCESS(rc))
10637 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10638 if (RT_FAILURE(rc))
10639 {
10640 rc = VERR_EM_INTERPRETER;
10641 break;
10642 }
10643 pMixedCtx->eip = 0;
10644 pMixedCtx->ip = aIretFrame[0];
10645 pMixedCtx->cs.Sel = aIretFrame[1];
10646 pMixedCtx->cs.ValidSel = aIretFrame[1];
10647 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10648 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10649 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10650 pMixedCtx->sp += sizeof(aIretFrame);
10651 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_SEGMENT_REGS | HM_CHANGED_GUEST_RSP
10652 | HM_CHANGED_GUEST_RFLAGS;
10653 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10654 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10655 break;
10656 }
10657
10658 case OP_INT:
10659 {
10660 uint16_t uVector = pDis->Param1.uValue & 0xff;
10661 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10662 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10663 break;
10664 }
10665
10666 case OP_INTO:
10667 {
10668 if (pMixedCtx->eflags.Bits.u1OF)
10669 {
10670 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10671 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10672 }
10673 break;
10674 }
10675
10676 default:
10677 {
10678 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10679 EMCODETYPE_SUPERVISOR);
10680 rc = VBOXSTRICTRC_VAL(rc2);
10681 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
10682 Log4(("#GP rc=%Rrc\n", rc));
10683 break;
10684 }
10685 }
10686 }
10687 else
10688 rc = VERR_EM_INTERPRETER;
10689
10690 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10691 ("#GP Unexpected rc=%Rrc\n", rc));
10692 return rc;
10693}
10694
10695
10696/**
10697 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10698 * the exception reported in the VMX transient structure back into the VM.
10699 *
10700 * @remarks Requires uExitIntrInfo in the VMX transient structure to be
10701 * up-to-date.
10702 */
10703static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10704{
10705 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10706
10707 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10708 hmR0VmxCheckExitDueToEventDelivery(). */
10709 int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10710 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10711 AssertRCReturn(rc, rc);
10712 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10713
10714 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10715 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10716 return VINF_SUCCESS;
10717}
10718
10719
10720/**
10721 * VM-exit exception handler for #PF (Page-fault exception).
10722 */
10723static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10724{
10725 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10726 PVM pVM = pVCpu->CTX_SUFF(pVM);
10727 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10728 rc |= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10729 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10730 AssertRCReturn(rc, rc);
10731
10732#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
10733 if (pVM->hm.s.fNestedPaging)
10734 {
10735 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10736 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10737 {
10738 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10739 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10740 0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10741 }
10742 else
10743 {
10744 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10745 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10746 Log4(("Pending #DF due to vectoring #PF. NP\n"));
10747 }
10748 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10749 return rc;
10750 }
10751#else
10752 Assert(!pVM->hm.s.fNestedPaging);
10753#endif
10754
10755 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10756 AssertRCReturn(rc, rc);
10757
10758 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10759 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10760
10761 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10762 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10763 (RTGCPTR)pVmxTransient->uExitQualification);
10764
10765 Log4(("#PF: rc=%Rrc\n", rc));
10766 if (rc == VINF_SUCCESS)
10767 {
10768 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10769 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
10770 * memory? We don't update the whole state here... */
10771 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10772 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10773 TRPMResetTrap(pVCpu);
10774 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10775 return rc;
10776 }
10777 else if (rc == VINF_EM_RAW_GUEST_TRAP)
10778 {
10779 if (!pVmxTransient->fVectoringPF)
10780 {
10781 /* It's a guest page fault and needs to be reflected to the guest. */
10782 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10783 TRPMResetTrap(pVCpu);
10784 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10785 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10786 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10787 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10788 }
10789 else
10790 {
10791 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10792 TRPMResetTrap(pVCpu);
10793 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10794 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10795 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10796 }
10797
10798 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10799 return VINF_SUCCESS;
10800 }
10801
10802 TRPMResetTrap(pVCpu);
10803 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10804 return rc;
10805}
10806
10807/** @} */
10808
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette