VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 48528

Last change on this file since 48528 was 48478, checked in by vboxsync, 12 years ago

HMVMX: Look at the right guest long mode indicator.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 449.6 KB
Line 
1/* $Id: HMVMXR0.cpp 48478 2013-09-13 16:26:24Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#define HMVMX_ALWAYS_SWAP_FPU_STATE
45#endif
46
47
48/*******************************************************************************
49* Defined Constants And Macros *
50*******************************************************************************/
51#if defined(RT_ARCH_AMD64)
52# define HMVMX_IS_64BIT_HOST_MODE() (true)
53typedef RTHCUINTREG HMVMXHCUINTREG;
54#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
55extern "C" uint32_t g_fVMXIs64bitHost;
56# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
57typedef uint64_t HMVMXHCUINTREG;
58#else
59# define HMVMX_IS_64BIT_HOST_MODE() (false)
60typedef RTHCUINTREG HMVMXHCUINTREG;
61#endif
62
63/** Use the function table. */
64#define HMVMX_USE_FUNCTION_TABLE
65
66/** Determine which tagged-TLB flush handler to use. */
67#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72/** @name Updated-guest-state flags.
73 * @{ */
74#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95 | HMVMX_UPDATED_GUEST_RSP \
96 | HMVMX_UPDATED_GUEST_RFLAGS \
97 | HMVMX_UPDATED_GUEST_CR0 \
98 | HMVMX_UPDATED_GUEST_CR3 \
99 | HMVMX_UPDATED_GUEST_CR4 \
100 | HMVMX_UPDATED_GUEST_GDTR \
101 | HMVMX_UPDATED_GUEST_IDTR \
102 | HMVMX_UPDATED_GUEST_LDTR \
103 | HMVMX_UPDATED_GUEST_TR \
104 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105 | HMVMX_UPDATED_GUEST_DEBUG \
106 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113 | HMVMX_UPDATED_GUEST_APIC_STATE)
114/** @} */
115
116/** @name
117 * Flags to skip redundant reads of some common VMCS fields that are not part of
118 * the guest-CPU state but are in the transient structure.
119 */
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126/** @} */
127
128/** @name
129 * States of the VMCS.
130 *
131 * This does not reflect all possible VMCS states but currently only those
132 * needed for maintaining the VMCS consistently even when thread-context hooks
133 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
134 */
135#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
136#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
137#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
138/** @} */
139
140/**
141 * Exception bitmap mask for real-mode guests (real-on-v86).
142 *
143 * We need to intercept all exceptions manually (except #PF). #NM is also
144 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
145 * even in real-mode if we have Nested Paging support.
146 */
147#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
148 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
149 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
150 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
151 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
153 | RT_BIT(X86_XCPT_XF))
154
155/**
156 * Exception bitmap mask for all contributory exceptions.
157 *
158 * Page fault is deliberately excluded here as it's conditional as to whether
159 * it's contributory or benign. Page faults are handled separately.
160 */
161#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
162 | RT_BIT(X86_XCPT_DE))
163
164/** Maximum VM-instruction error number. */
165#define HMVMX_INSTR_ERROR_MAX 28
166
167/** Profiling macro. */
168#ifdef HM_PROFILE_EXIT_DISPATCH
169# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171#else
172# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174#endif
175
176/** Assert that preemption is disabled or covered by thread-context hooks. */
177#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
178 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180/** Assert that we haven't migrated CPUs when thread-context hooks are not
181 * used. */
182#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
183 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184 ("Illegal migration! Entered on CPU %u Current %u\n", \
185 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187/*******************************************************************************
188* Structures and Typedefs *
189*******************************************************************************/
190/**
191 * VMX transient state.
192 *
193 * A state structure for holding miscellaneous information across
194 * VMX non-root operation and restored after the transition.
195 */
196typedef struct VMXTRANSIENT
197{
198 /** The host's rflags/eflags. */
199 RTCCUINTREG uEflags;
200#if HC_ARCH_BITS == 32
201 uint32_t u32Alignment0;
202#endif
203 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
204 uint64_t u64LStarMsr;
205 /** The guest's TPR value used for TPR shadowing. */
206 uint8_t u8GuestTpr;
207 /** Alignment. */
208 uint8_t abAlignment0[7];
209
210 /** The basic VM-exit reason. */
211 uint16_t uExitReason;
212 /** Alignment. */
213 uint16_t u16Alignment0;
214 /** The VM-exit interruption error code. */
215 uint32_t uExitIntrErrorCode;
216 /** The VM-exit exit qualification. */
217 uint64_t uExitQualification;
218
219 /** The VM-exit interruption-information field. */
220 uint32_t uExitIntrInfo;
221 /** The VM-exit instruction-length field. */
222 uint32_t cbInstr;
223 /** The VM-exit instruction-information field. */
224 union
225 {
226 /** Plain unsigned int representation. */
227 uint32_t u;
228 /** INS and OUTS information. */
229 struct
230 {
231 uint32_t u6Reserved0 : 6;
232 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
233 uint32_t u3AddrSize : 3;
234 uint32_t u5Reserved1 : 5;
235 /** The segment register (X86_SREG_XXX). */
236 uint32_t iSegReg : 3;
237 uint32_t uReserved2 : 14;
238 } StrIo;
239 } ExitInstrInfo;
240 /** Whether the VM-entry failed or not. */
241 bool fVMEntryFailed;
242 /** Alignment. */
243 uint8_t abAlignment1[3];
244
245 /** The VM-entry interruption-information field. */
246 uint32_t uEntryIntrInfo;
247 /** The VM-entry exception error code field. */
248 uint32_t uEntryXcptErrorCode;
249 /** The VM-entry instruction length field. */
250 uint32_t cbEntryInstr;
251
252 /** IDT-vectoring information field. */
253 uint32_t uIdtVectoringInfo;
254 /** IDT-vectoring error code. */
255 uint32_t uIdtVectoringErrorCode;
256
257 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
258 uint32_t fVmcsFieldsRead;
259 /** Whether TSC-offsetting should be setup before VM-entry. */
260 bool fUpdateTscOffsettingAndPreemptTimer;
261 /** Whether the VM-exit was caused by a page-fault during delivery of a
262 * contributory exception or a page-fault. */
263 bool fVectoringPF;
264} VMXTRANSIENT;
265AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
266AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
267AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
268AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
269/** Pointer to VMX transient state. */
270typedef VMXTRANSIENT *PVMXTRANSIENT;
271
272
273/**
274 * MSR-bitmap read permissions.
275 */
276typedef enum VMXMSREXITREAD
277{
278 /** Reading this MSR causes a VM-exit. */
279 VMXMSREXIT_INTERCEPT_READ = 0xb,
280 /** Reading this MSR does not cause a VM-exit. */
281 VMXMSREXIT_PASSTHRU_READ
282} VMXMSREXITREAD;
283
284/**
285 * MSR-bitmap write permissions.
286 */
287typedef enum VMXMSREXITWRITE
288{
289 /** Writing to this MSR causes a VM-exit. */
290 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
291 /** Writing to this MSR does not cause a VM-exit. */
292 VMXMSREXIT_PASSTHRU_WRITE
293} VMXMSREXITWRITE;
294
295/**
296 * VMX VM-exit handler.
297 *
298 * @returns VBox status code.
299 * @param pVCpu Pointer to the VMCPU.
300 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
301 * out-of-sync. Make sure to update the required
302 * fields before using them.
303 * @param pVmxTransient Pointer to the VMX-transient structure.
304 */
305#ifndef HMVMX_USE_FUNCTION_TABLE
306typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
307#else
308typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309/** Pointer to VM-exit handler. */
310typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
311#endif
312
313
314/*******************************************************************************
315* Internal Functions *
316*******************************************************************************/
317static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
318static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
319static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
320static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
321 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
322#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
323static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
324#endif
325#ifndef HMVMX_USE_FUNCTION_TABLE
326DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
327# define HMVMX_EXIT_DECL static int
328#else
329# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
330#endif
331
332/** @name VM-exit handlers.
333 * @{
334 */
335static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
336static FNVMXEXITHANDLER hmR0VmxExitExtInt;
337static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
338static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
339static FNVMXEXITHANDLER hmR0VmxExitSipi;
340static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
341static FNVMXEXITHANDLER hmR0VmxExitSmi;
342static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
343static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
344static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
345static FNVMXEXITHANDLER hmR0VmxExitCpuid;
346static FNVMXEXITHANDLER hmR0VmxExitGetsec;
347static FNVMXEXITHANDLER hmR0VmxExitHlt;
348static FNVMXEXITHANDLER hmR0VmxExitInvd;
349static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
350static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
351static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
352static FNVMXEXITHANDLER hmR0VmxExitRsm;
353static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
354static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
355static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
356static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
357static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
358static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
359static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
360static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
361static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
362static FNVMXEXITHANDLER hmR0VmxExitMwait;
363static FNVMXEXITHANDLER hmR0VmxExitMtf;
364static FNVMXEXITHANDLER hmR0VmxExitMonitor;
365static FNVMXEXITHANDLER hmR0VmxExitPause;
366static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
367static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
368static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
369static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
370static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
371static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
372static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
373static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
374static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
375static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
376static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
377static FNVMXEXITHANDLER hmR0VmxExitRdrand;
378static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
379/** @} */
380
381static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
382static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
383static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
384static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
385static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
386static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
387static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
388static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
389
390/*******************************************************************************
391* Global Variables *
392*******************************************************************************/
393#ifdef HMVMX_USE_FUNCTION_TABLE
394
395/**
396 * VMX_EXIT dispatch table.
397 */
398static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
399{
400 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
401 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
402 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
403 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
404 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
405 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
406 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
407 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
408 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
409 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
410 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
411 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
412 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
413 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
414 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
415 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
416 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
417 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
418 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
419 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
420 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
421 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
422 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
423 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
424 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
425 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
426 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
427 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
428 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
429 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
430 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
431 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
432 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
433 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
434 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
435 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
436 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
437 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
438 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
439 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
440 /* 40 UNDEFINED */ hmR0VmxExitPause,
441 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
442 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
443 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
444 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
445 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
446 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
447 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
448 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
449 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
450 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
451 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
452 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
453 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
454 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
455 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
456 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
457 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
458 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
459 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
460};
461#endif /* HMVMX_USE_FUNCTION_TABLE */
462
463#ifdef VBOX_STRICT
464static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
465{
466 /* 0 */ "(Not Used)",
467 /* 1 */ "VMCALL executed in VMX root operation.",
468 /* 2 */ "VMCLEAR with invalid physical address.",
469 /* 3 */ "VMCLEAR with VMXON pointer.",
470 /* 4 */ "VMLAUNCH with non-clear VMCS.",
471 /* 5 */ "VMRESUME with non-launched VMCS.",
472 /* 6 */ "VMRESUME after VMXOFF",
473 /* 7 */ "VM entry with invalid control fields.",
474 /* 8 */ "VM entry with invalid host state fields.",
475 /* 9 */ "VMPTRLD with invalid physical address.",
476 /* 10 */ "VMPTRLD with VMXON pointer.",
477 /* 11 */ "VMPTRLD with incorrect revision identifier.",
478 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
479 /* 13 */ "VMWRITE to read-only VMCS component.",
480 /* 14 */ "(Not Used)",
481 /* 15 */ "VMXON executed in VMX root operation.",
482 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
483 /* 17 */ "VM entry with non-launched executing VMCS.",
484 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
485 /* 19 */ "VMCALL with non-clear VMCS.",
486 /* 20 */ "VMCALL with invalid VM-exit control fields.",
487 /* 21 */ "(Not Used)",
488 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
489 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
490 /* 24 */ "VMCALL with invalid SMM-monitor features.",
491 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
492 /* 26 */ "VM entry with events blocked by MOV SS.",
493 /* 27 */ "(Not Used)",
494 /* 28 */ "Invalid operand to INVEPT/INVVPID."
495};
496#endif /* VBOX_STRICT */
497
498
499
500/**
501 * Updates the VM's last error record. If there was a VMX instruction error,
502 * reads the error data from the VMCS and updates VCPU's last error record as
503 * well.
504 *
505 * @param pVM Pointer to the VM.
506 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
507 * VERR_VMX_UNABLE_TO_START_VM or
508 * VERR_VMX_INVALID_VMCS_FIELD).
509 * @param rc The error code.
510 */
511static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
512{
513 AssertPtr(pVM);
514 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
515 || rc == VERR_VMX_UNABLE_TO_START_VM)
516 {
517 AssertPtrReturnVoid(pVCpu);
518 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
519 }
520 pVM->hm.s.lLastError = rc;
521}
522
523
524/**
525 * Reads the VM-entry interruption-information field from the VMCS into the VMX
526 * transient structure.
527 *
528 * @returns VBox status code.
529 * @param pVmxTransient Pointer to the VMX transient structure.
530 *
531 * @remarks No-long-jump zone!!!
532 */
533DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
534{
535 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
536 AssertRCReturn(rc, rc);
537 return VINF_SUCCESS;
538}
539
540
541/**
542 * Reads the VM-entry exception error code field from the VMCS into
543 * the VMX transient structure.
544 *
545 * @returns VBox status code.
546 * @param pVmxTransient Pointer to the VMX transient structure.
547 *
548 * @remarks No-long-jump zone!!!
549 */
550DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
551{
552 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
553 AssertRCReturn(rc, rc);
554 return VINF_SUCCESS;
555}
556
557
558/**
559 * Reads the VM-entry exception error code field from the VMCS into
560 * the VMX transient structure.
561 *
562 * @returns VBox status code.
563 * @param pVCpu Pointer to the VMCPU.
564 * @param pVmxTransient Pointer to the VMX transient structure.
565 *
566 * @remarks No-long-jump zone!!!
567 */
568DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
569{
570 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
571 AssertRCReturn(rc, rc);
572 return VINF_SUCCESS;
573}
574
575
576/**
577 * Reads the VM-exit interruption-information field from the VMCS into the VMX
578 * transient structure.
579 *
580 * @returns VBox status code.
581 * @param pVCpu Pointer to the VMCPU.
582 * @param pVmxTransient Pointer to the VMX transient structure.
583 */
584DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
585{
586 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
587 {
588 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
589 AssertRCReturn(rc, rc);
590 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
591 }
592 return VINF_SUCCESS;
593}
594
595
596/**
597 * Reads the VM-exit interruption error code from the VMCS into the VMX
598 * transient structure.
599 *
600 * @returns VBox status code.
601 * @param pVCpu Pointer to the VMCPU.
602 * @param pVmxTransient Pointer to the VMX transient structure.
603 */
604DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
605{
606 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
607 {
608 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
609 AssertRCReturn(rc, rc);
610 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
611 }
612 return VINF_SUCCESS;
613}
614
615
616/**
617 * Reads the VM-exit instruction length field from the VMCS into the VMX
618 * transient structure.
619 *
620 * @returns VBox status code.
621 * @param pVCpu Pointer to the VMCPU.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 */
624DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
625{
626 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
627 {
628 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
629 AssertRCReturn(rc, rc);
630 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
631 }
632 return VINF_SUCCESS;
633}
634
635
636/**
637 * Reads the VM-exit instruction-information field from the VMCS into
638 * the VMX transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVCpu The cross context per CPU structure.
642 * @param pVmxTransient Pointer to the VMX transient structure.
643 */
644DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
645{
646 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
647 {
648 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
649 AssertRCReturn(rc, rc);
650 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
651 }
652 return VINF_SUCCESS;
653}
654
655
656/**
657 * Reads the exit qualification from the VMCS into the VMX transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVCpu Pointer to the VMCPU.
661 * @param pVmxTransient Pointer to the VMX transient structure.
662 */
663DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
664{
665 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
666 {
667 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
668 AssertRCReturn(rc, rc);
669 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
670 }
671 return VINF_SUCCESS;
672}
673
674
675/**
676 * Reads the IDT-vectoring information field from the VMCS into the VMX
677 * transient structure.
678 *
679 * @returns VBox status code.
680 * @param pVmxTransient Pointer to the VMX transient structure.
681 *
682 * @remarks No-long-jump zone!!!
683 */
684DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
685{
686 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
687 {
688 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
689 AssertRCReturn(rc, rc);
690 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
691 }
692 return VINF_SUCCESS;
693}
694
695
696/**
697 * Reads the IDT-vectoring error code from the VMCS into the VMX
698 * transient structure.
699 *
700 * @returns VBox status code.
701 * @param pVmxTransient Pointer to the VMX transient structure.
702 */
703DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
704{
705 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
706 {
707 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
708 AssertRCReturn(rc, rc);
709 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
710 }
711 return VINF_SUCCESS;
712}
713
714
715/**
716 * Enters VMX root mode operation on the current CPU.
717 *
718 * @returns VBox status code.
719 * @param pVM Pointer to the VM (optional, can be NULL, after
720 * a resume).
721 * @param HCPhysCpuPage Physical address of the VMXON region.
722 * @param pvCpuPage Pointer to the VMXON region.
723 */
724static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
725{
726 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
727 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
728 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
729
730 if (pVM)
731 {
732 /* Write the VMCS revision dword to the VMXON region. */
733 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
734 }
735
736 /* Enable the VMX bit in CR4 if necessary. */
737 RTCCUINTREG uCr4 = ASMGetCR4();
738 if (!(uCr4 & X86_CR4_VMXE))
739 ASMSetCR4(uCr4 | X86_CR4_VMXE);
740
741 /* Enter VMX root mode. */
742 int rc = VMXEnable(HCPhysCpuPage);
743 if (RT_FAILURE(rc))
744 ASMSetCR4(uCr4);
745
746 return rc;
747}
748
749
750/**
751 * Exits VMX root mode operation on the current CPU.
752 *
753 * @returns VBox status code.
754 */
755static int hmR0VmxLeaveRootMode(void)
756{
757 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
758
759 /* If we're for some reason not in VMX root mode, then don't leave it. */
760 RTCCUINTREG uHostCR4 = ASMGetCR4();
761 if (uHostCR4 & X86_CR4_VMXE)
762 {
763 /* Exit VMX root mode and clear the VMX bit in CR4. */
764 VMXDisable();
765 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
766 return VINF_SUCCESS;
767 }
768
769 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
770}
771
772
773/**
774 * Allocates and maps one physically contiguous page. The allocated page is
775 * zero'd out. (Used by various VT-x structures).
776 *
777 * @returns IPRT status code.
778 * @param pMemObj Pointer to the ring-0 memory object.
779 * @param ppVirt Where to store the virtual address of the
780 * allocation.
781 * @param pPhys Where to store the physical address of the
782 * allocation.
783 */
784DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
785{
786 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
787 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
788 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
789
790 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
791 if (RT_FAILURE(rc))
792 return rc;
793 *ppVirt = RTR0MemObjAddress(*pMemObj);
794 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
795 ASMMemZero32(*ppVirt, PAGE_SIZE);
796 return VINF_SUCCESS;
797}
798
799
800/**
801 * Frees and unmaps an allocated physical page.
802 *
803 * @param pMemObj Pointer to the ring-0 memory object.
804 * @param ppVirt Where to re-initialize the virtual address of
805 * allocation as 0.
806 * @param pHCPhys Where to re-initialize the physical address of the
807 * allocation as 0.
808 */
809DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
810{
811 AssertPtr(pMemObj);
812 AssertPtr(ppVirt);
813 AssertPtr(pHCPhys);
814 if (*pMemObj != NIL_RTR0MEMOBJ)
815 {
816 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
817 AssertRC(rc);
818 *pMemObj = NIL_RTR0MEMOBJ;
819 *ppVirt = 0;
820 *pHCPhys = 0;
821 }
822}
823
824
825/**
826 * Worker function to free VT-x related structures.
827 *
828 * @returns IPRT status code.
829 * @param pVM Pointer to the VM.
830 */
831static void hmR0VmxStructsFree(PVM pVM)
832{
833 for (VMCPUID i = 0; i < pVM->cCpus; i++)
834 {
835 PVMCPU pVCpu = &pVM->aCpus[i];
836 AssertPtr(pVCpu);
837
838#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
839 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
840 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
841#endif
842
843 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
844 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
845
846 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
847 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
848 }
849
850 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
851#ifdef VBOX_WITH_CRASHDUMP_MAGIC
852 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
853#endif
854}
855
856
857/**
858 * Worker function to allocate VT-x related VM structures.
859 *
860 * @returns IPRT status code.
861 * @param pVM Pointer to the VM.
862 */
863static int hmR0VmxStructsAlloc(PVM pVM)
864{
865 /*
866 * Initialize members up-front so we can cleanup properly on allocation failure.
867 */
868#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
869 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
870 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
871 pVM->hm.s.vmx.HCPhys##a_Name = 0;
872
873#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
874 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
875 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
876 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
877
878#ifdef VBOX_WITH_CRASHDUMP_MAGIC
879 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
880#endif
881 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
882
883 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
884 for (VMCPUID i = 0; i < pVM->cCpus; i++)
885 {
886 PVMCPU pVCpu = &pVM->aCpus[i];
887 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
888 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
889 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
890#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
891 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
892 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
893#endif
894 }
895#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
896#undef VMXLOCAL_INIT_VM_MEMOBJ
897
898 /*
899 * Allocate all the VT-x structures.
900 */
901 int rc = VINF_SUCCESS;
902#ifdef VBOX_WITH_CRASHDUMP_MAGIC
903 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
904 if (RT_FAILURE(rc))
905 goto cleanup;
906 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
907 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
908#endif
909
910 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
911 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
912 {
913 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
914 &pVM->hm.s.vmx.HCPhysApicAccess);
915 if (RT_FAILURE(rc))
916 goto cleanup;
917 }
918
919 /*
920 * Initialize per-VCPU VT-x structures.
921 */
922 for (VMCPUID i = 0; i < pVM->cCpus; i++)
923 {
924 PVMCPU pVCpu = &pVM->aCpus[i];
925 AssertPtr(pVCpu);
926
927 /* Allocate the VM control structure (VMCS). */
928 AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
929 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
930 if (RT_FAILURE(rc))
931 goto cleanup;
932
933 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
934 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
935 {
936 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
937 &pVCpu->hm.s.vmx.HCPhysVirtApic);
938 if (RT_FAILURE(rc))
939 goto cleanup;
940 }
941
942 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
943 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
944 {
945 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
946 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
947 if (RT_FAILURE(rc))
948 goto cleanup;
949 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
950 }
951
952#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
953 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
954 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
955 if (RT_FAILURE(rc))
956 goto cleanup;
957
958 /* Allocate the VM-exit MSR-load page for the host MSRs. */
959 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
960 if (RT_FAILURE(rc))
961 goto cleanup;
962#endif
963 }
964
965 return VINF_SUCCESS;
966
967cleanup:
968 hmR0VmxStructsFree(pVM);
969 return rc;
970}
971
972
973/**
974 * Does global VT-x initialization (called during module initialization).
975 *
976 * @returns VBox status code.
977 */
978VMMR0DECL(int) VMXR0GlobalInit(void)
979{
980#ifdef HMVMX_USE_FUNCTION_TABLE
981 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
982# ifdef VBOX_STRICT
983 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
984 Assert(g_apfnVMExitHandlers[i]);
985# endif
986#endif
987 return VINF_SUCCESS;
988}
989
990
991/**
992 * Does global VT-x termination (called during module termination).
993 */
994VMMR0DECL(void) VMXR0GlobalTerm()
995{
996 /* Nothing to do currently. */
997}
998
999
1000/**
1001 * Sets up and activates VT-x on the current CPU.
1002 *
1003 * @returns VBox status code.
1004 * @param pCpu Pointer to the global CPU info struct.
1005 * @param pVM Pointer to the VM (can be NULL after a host resume
1006 * operation).
1007 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1008 * fEnabledByHost is true).
1009 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1010 * @a fEnabledByHost is true).
1011 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1012 * enable VT-x on the host.
1013 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1014 */
1015VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1016 void *pvMsrs)
1017{
1018 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1019 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1020 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1021
1022 /* Enable VT-x if it's not already enabled by the host. */
1023 if (!fEnabledByHost)
1024 {
1025 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1026 if (RT_FAILURE(rc))
1027 return rc;
1028 }
1029
1030 /*
1031 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1032 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1033 */
1034 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1035 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1036 {
1037 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1038 pCpu->fFlushAsidBeforeUse = false;
1039 }
1040 else
1041 pCpu->fFlushAsidBeforeUse = true;
1042
1043 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1044 ++pCpu->cTlbFlushes;
1045
1046 return VINF_SUCCESS;
1047}
1048
1049
1050/**
1051 * Deactivates VT-x on the current CPU.
1052 *
1053 * @returns VBox status code.
1054 * @param pCpu Pointer to the global CPU info struct.
1055 * @param pvCpuPage Pointer to the VMXON region.
1056 * @param HCPhysCpuPage Physical address of the VMXON region.
1057 *
1058 * @remarks This function should never be called when SUPR0EnableVTx() or
1059 * similar was used to enable VT-x on the host.
1060 */
1061VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1062{
1063 NOREF(pCpu);
1064 NOREF(pvCpuPage);
1065 NOREF(HCPhysCpuPage);
1066
1067 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1068 return hmR0VmxLeaveRootMode();
1069}
1070
1071
1072/**
1073 * Sets the permission bits for the specified MSR in the MSR bitmap.
1074 *
1075 * @param pVCpu Pointer to the VMCPU.
1076 * @param uMSR The MSR value.
1077 * @param enmRead Whether reading this MSR causes a VM-exit.
1078 * @param enmWrite Whether writing this MSR causes a VM-exit.
1079 */
1080static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1081{
1082 int32_t iBit;
1083 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1084
1085 /*
1086 * Layout:
1087 * 0x000 - 0x3ff - Low MSR read bits
1088 * 0x400 - 0x7ff - High MSR read bits
1089 * 0x800 - 0xbff - Low MSR write bits
1090 * 0xc00 - 0xfff - High MSR write bits
1091 */
1092 if (uMsr <= 0x00001FFF)
1093 iBit = uMsr;
1094 else if ( uMsr >= 0xC0000000
1095 && uMsr <= 0xC0001FFF)
1096 {
1097 iBit = (uMsr - 0xC0000000);
1098 pbMsrBitmap += 0x400;
1099 }
1100 else
1101 {
1102 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1103 return;
1104 }
1105
1106 Assert(iBit <= 0x1fff);
1107 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1108 ASMBitSet(pbMsrBitmap, iBit);
1109 else
1110 ASMBitClear(pbMsrBitmap, iBit);
1111
1112 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1113 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1114 else
1115 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1116}
1117
1118
1119/**
1120 * Flushes the TLB using EPT.
1121 *
1122 * @returns VBox status code.
1123 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1124 * enmFlush).
1125 * @param enmFlush Type of flush.
1126 *
1127 * @remarks Caller is responsible for making sure this function is called only
1128 * when NestedPaging is supported and providing @a enmFlush that is
1129 * supported by the CPU.
1130 */
1131static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1132{
1133 uint64_t au64Descriptor[2];
1134 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1135 au64Descriptor[0] = 0;
1136 else
1137 {
1138 Assert(pVCpu);
1139 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1140 }
1141 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1142
1143 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1144 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1145 rc));
1146 if ( RT_SUCCESS(rc)
1147 && pVCpu)
1148 {
1149 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1150 }
1151}
1152
1153
1154/**
1155 * Flushes the TLB using VPID.
1156 *
1157 * @returns VBox status code.
1158 * @param pVM Pointer to the VM.
1159 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1160 * enmFlush).
1161 * @param enmFlush Type of flush.
1162 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1163 * on @a enmFlush).
1164 */
1165static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1166{
1167 AssertPtr(pVM);
1168 Assert(pVM->hm.s.vmx.fVpid);
1169
1170 uint64_t au64Descriptor[2];
1171 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1172 {
1173 au64Descriptor[0] = 0;
1174 au64Descriptor[1] = 0;
1175 }
1176 else
1177 {
1178 AssertPtr(pVCpu);
1179 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1180 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1181 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1182 au64Descriptor[1] = GCPtr;
1183 }
1184
1185 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1186 AssertMsg(rc == VINF_SUCCESS,
1187 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1188 if ( RT_SUCCESS(rc)
1189 && pVCpu)
1190 {
1191 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1192 }
1193}
1194
1195
1196/**
1197 * Invalidates a guest page by guest virtual address. Only relevant for
1198 * EPT/VPID, otherwise there is nothing really to invalidate.
1199 *
1200 * @returns VBox status code.
1201 * @param pVM Pointer to the VM.
1202 * @param pVCpu Pointer to the VMCPU.
1203 * @param GCVirt Guest virtual address of the page to invalidate.
1204 */
1205VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1206{
1207 AssertPtr(pVM);
1208 AssertPtr(pVCpu);
1209 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1210
1211 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1212 if (!fFlushPending)
1213 {
1214 /*
1215 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1216 * See @bugref{6043} and @bugref{6177}.
1217 *
1218 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1219 * function maybe called in a loop with individual addresses.
1220 */
1221 if (pVM->hm.s.vmx.fVpid)
1222 {
1223 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1224 {
1225 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1226 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1227 }
1228 else
1229 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1230 }
1231 else if (pVM->hm.s.fNestedPaging)
1232 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1233 }
1234
1235 return VINF_SUCCESS;
1236}
1237
1238
1239/**
1240 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1241 * otherwise there is nothing really to invalidate.
1242 *
1243 * @returns VBox status code.
1244 * @param pVM Pointer to the VM.
1245 * @param pVCpu Pointer to the VMCPU.
1246 * @param GCPhys Guest physical address of the page to invalidate.
1247 */
1248VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1249{
1250 LogFlowFunc(("%RGp\n", GCPhys));
1251
1252 /*
1253 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1254 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1255 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1256 */
1257 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1258 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1259 return VINF_SUCCESS;
1260}
1261
1262
1263/**
1264 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1265 * case where neither EPT nor VPID is supported by the CPU.
1266 *
1267 * @param pVM Pointer to the VM.
1268 * @param pVCpu Pointer to the VMCPU.
1269 * @param pCpu Pointer to the global HM struct.
1270 *
1271 * @remarks Called with interrupts disabled.
1272 */
1273static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1274{
1275 AssertPtr(pVCpu);
1276 AssertPtr(pCpu);
1277 NOREF(pVM);
1278
1279 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1280 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1281
1282 pVCpu->hm.s.TlbShootdown.cPages = 0;
1283 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1284 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1285 pVCpu->hm.s.fForceTLBFlush = false;
1286 return;
1287}
1288
1289
1290/**
1291 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1292 *
1293 * @param pVM Pointer to the VM.
1294 * @param pVCpu Pointer to the VMCPU.
1295 * @param pCpu Pointer to the global HM CPU struct.
1296 * @remarks All references to "ASID" in this function pertains to "VPID" in
1297 * Intel's nomenclature. The reason is, to avoid confusion in compare
1298 * statements since the host-CPU copies are named "ASID".
1299 *
1300 * @remarks Called with interrupts disabled.
1301 */
1302static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1303{
1304#ifdef VBOX_WITH_STATISTICS
1305 bool fTlbFlushed = false;
1306# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1307# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1308 if (!fTlbFlushed) \
1309 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1310 } while (0)
1311#else
1312# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1313# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1314#endif
1315
1316 AssertPtr(pVM);
1317 AssertPtr(pCpu);
1318 AssertPtr(pVCpu);
1319 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1320 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1321 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1322
1323
1324 /*
1325 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1326 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1327 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1328 */
1329 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1330 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1331 {
1332 ++pCpu->uCurrentAsid;
1333 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1334 {
1335 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1336 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1337 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1338 }
1339
1340 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1341 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1342 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1343
1344 /*
1345 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1346 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1347 */
1348 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1349 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1350 HMVMX_SET_TAGGED_TLB_FLUSHED();
1351 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1352 }
1353
1354 /* Check for explicit TLB shootdowns. */
1355 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1356 {
1357 /*
1358 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1359 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1360 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1361 * but not guest-physical mappings.
1362 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1363 */
1364 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1365 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1366 HMVMX_SET_TAGGED_TLB_FLUSHED();
1367 }
1368
1369 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1370 * not be executed. See hmQueueInvlPage() where it is commented
1371 * out. Support individual entry flushing someday. */
1372 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1373 {
1374 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1375
1376 /*
1377 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1378 * as supported by the CPU.
1379 */
1380 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1381 {
1382 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1383 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1384 }
1385 else
1386 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1387
1388 HMVMX_SET_TAGGED_TLB_FLUSHED();
1389 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1390 }
1391
1392 pVCpu->hm.s.TlbShootdown.cPages = 0;
1393 pVCpu->hm.s.fForceTLBFlush = false;
1394
1395 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1396
1397 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1398 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1399 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1400 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1401 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1402 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1403 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1404 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1405
1406 /* Update VMCS with the VPID. */
1407 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1408 AssertRC(rc);
1409
1410#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1411}
1412
1413
1414/**
1415 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1416 *
1417 * @returns VBox status code.
1418 * @param pVM Pointer to the VM.
1419 * @param pVCpu Pointer to the VMCPU.
1420 * @param pCpu Pointer to the global HM CPU struct.
1421 *
1422 * @remarks Called with interrupts disabled.
1423 */
1424static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1425{
1426 AssertPtr(pVM);
1427 AssertPtr(pVCpu);
1428 AssertPtr(pCpu);
1429 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1430 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1431
1432 /*
1433 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1434 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1435 */
1436 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1437 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1438 {
1439 pVCpu->hm.s.fForceTLBFlush = true;
1440 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1441 }
1442
1443 /* Check for explicit TLB shootdown flushes. */
1444 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1445 {
1446 pVCpu->hm.s.fForceTLBFlush = true;
1447 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1448 }
1449
1450 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1451 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1452
1453 if (pVCpu->hm.s.fForceTLBFlush)
1454 {
1455 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1456 pVCpu->hm.s.fForceTLBFlush = false;
1457 }
1458 else
1459 {
1460 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1461 * not be executed. See hmQueueInvlPage() where it is commented
1462 * out. Support individual entry flushing someday. */
1463 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1464 {
1465 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1466 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1467 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1468 }
1469 else
1470 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1471 }
1472
1473 pVCpu->hm.s.TlbShootdown.cPages = 0;
1474 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1475}
1476
1477
1478/**
1479 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1480 *
1481 * @returns VBox status code.
1482 * @param pVM Pointer to the VM.
1483 * @param pVCpu Pointer to the VMCPU.
1484 * @param pCpu Pointer to the global HM CPU struct.
1485 *
1486 * @remarks Called with interrupts disabled.
1487 */
1488static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1489{
1490 AssertPtr(pVM);
1491 AssertPtr(pVCpu);
1492 AssertPtr(pCpu);
1493 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1494 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1495
1496 /*
1497 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1498 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1499 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1500 */
1501 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1502 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1503 {
1504 pVCpu->hm.s.fForceTLBFlush = true;
1505 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1506 }
1507
1508 /* Check for explicit TLB shootdown flushes. */
1509 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1510 {
1511 /*
1512 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1513 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1514 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1515 */
1516 pVCpu->hm.s.fForceTLBFlush = true;
1517 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1518 }
1519
1520 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1521 if (pVCpu->hm.s.fForceTLBFlush)
1522 {
1523 ++pCpu->uCurrentAsid;
1524 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1525 {
1526 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1527 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1528 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1529 }
1530
1531 pVCpu->hm.s.fForceTLBFlush = false;
1532 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1533 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1534 if (pCpu->fFlushAsidBeforeUse)
1535 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1536 }
1537 else
1538 {
1539 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1540 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1541 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1542 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1543
1544 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1545 * not be executed. See hmQueueInvlPage() where it is commented
1546 * out. Support individual entry flushing someday. */
1547 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1548 {
1549 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1550 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1551 {
1552 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1553 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1554 }
1555 else
1556 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1557 }
1558 else
1559 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1560 }
1561
1562 pVCpu->hm.s.TlbShootdown.cPages = 0;
1563 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1564
1565 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1566 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1567 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1568 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1569 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1570 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1571
1572 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1573 AssertRC(rc);
1574}
1575
1576
1577/**
1578 * Flushes the guest TLB entry based on CPU capabilities.
1579 *
1580 * @param pVCpu Pointer to the VMCPU.
1581 * @param pCpu Pointer to the global HM CPU struct.
1582 */
1583DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1584{
1585 PVM pVM = pVCpu->CTX_SUFF(pVM);
1586 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1587 {
1588 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1589 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1590 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1591 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1592 default:
1593 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1594 break;
1595 }
1596}
1597
1598
1599/**
1600 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1601 * TLB entries from the host TLB before VM-entry.
1602 *
1603 * @returns VBox status code.
1604 * @param pVM Pointer to the VM.
1605 */
1606static int hmR0VmxSetupTaggedTlb(PVM pVM)
1607{
1608 /*
1609 * Determine optimal flush type for Nested Paging.
1610 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1611 * guest execution (see hmR3InitFinalizeR0()).
1612 */
1613 if (pVM->hm.s.fNestedPaging)
1614 {
1615 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1616 {
1617 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1618 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1619 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1620 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1621 else
1622 {
1623 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1624 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1625 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1626 }
1627
1628 /* Make sure the write-back cacheable memory type for EPT is supported. */
1629 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1630 {
1631 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1632 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1633 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1634 }
1635 }
1636 else
1637 {
1638 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1639 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1640 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1641 }
1642 }
1643
1644 /*
1645 * Determine optimal flush type for VPID.
1646 */
1647 if (pVM->hm.s.vmx.fVpid)
1648 {
1649 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1650 {
1651 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1652 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1653 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1654 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1655 else
1656 {
1657 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1658 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1659 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1660 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1661 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1662 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1663 pVM->hm.s.vmx.fVpid = false;
1664 }
1665 }
1666 else
1667 {
1668 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1669 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1670 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1671 pVM->hm.s.vmx.fVpid = false;
1672 }
1673 }
1674
1675 /*
1676 * Setup the handler for flushing tagged-TLBs.
1677 */
1678 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1679 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1680 else if (pVM->hm.s.fNestedPaging)
1681 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1682 else if (pVM->hm.s.vmx.fVpid)
1683 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1684 else
1685 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1686 return VINF_SUCCESS;
1687}
1688
1689
1690/**
1691 * Sets up pin-based VM-execution controls in the VMCS.
1692 *
1693 * @returns VBox status code.
1694 * @param pVM Pointer to the VM.
1695 * @param pVCpu Pointer to the VMCPU.
1696 */
1697static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1698{
1699 AssertPtr(pVM);
1700 AssertPtr(pVCpu);
1701
1702 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1703 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1704
1705 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1706 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1707 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1708
1709 /* Enable the VMX preemption timer. */
1710 if (pVM->hm.s.vmx.fUsePreemptTimer)
1711 {
1712 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1713 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1714 }
1715
1716 if ((val & zap) != val)
1717 {
1718 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1719 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1720 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1721 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1722 }
1723
1724 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1725 AssertRCReturn(rc, rc);
1726
1727 /* Update VCPU with the currently set pin-based VM-execution controls. */
1728 pVCpu->hm.s.vmx.u32PinCtls = val;
1729 return rc;
1730}
1731
1732
1733/**
1734 * Sets up processor-based VM-execution controls in the VMCS.
1735 *
1736 * @returns VBox status code.
1737 * @param pVM Pointer to the VM.
1738 * @param pVMCPU Pointer to the VMCPU.
1739 */
1740static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1741{
1742 AssertPtr(pVM);
1743 AssertPtr(pVCpu);
1744
1745 int rc = VERR_INTERNAL_ERROR_5;
1746 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1747 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1748
1749 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1750 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1751 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1752 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1753 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1754 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1755 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1756
1757 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1758 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1759 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1760 {
1761 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1762 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1763 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1764 }
1765
1766 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1767 if (!pVM->hm.s.fNestedPaging)
1768 {
1769 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1770 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1771 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1772 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1773 }
1774
1775 /* Use TPR shadowing if supported by the CPU. */
1776 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1777 {
1778 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1779 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1780 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1781 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1782 AssertRCReturn(rc, rc);
1783
1784 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1785 /* CR8 writes causes a VM-exit based on TPR threshold. */
1786 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1787 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1788 }
1789 else
1790 {
1791 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1792 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1793 }
1794
1795 /* Use MSR-bitmaps if supported by the CPU. */
1796 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1797 {
1798 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1799
1800 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1801 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1802 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1803 AssertRCReturn(rc, rc);
1804
1805 /*
1806 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1807 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1808 */
1809 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1810 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1811 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1812 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1813 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1814 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1815 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1816 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1817 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1818 }
1819
1820 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1821 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1822 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1823
1824 if ((val & zap) != val)
1825 {
1826 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1827 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1828 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1829 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1830 }
1831
1832 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1833 AssertRCReturn(rc, rc);
1834
1835 /* Update VCPU with the currently set processor-based VM-execution controls. */
1836 pVCpu->hm.s.vmx.u32ProcCtls = val;
1837
1838 /*
1839 * Secondary processor-based VM-execution controls.
1840 */
1841 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1842 {
1843 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1844 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1845
1846 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1847 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1848
1849 if (pVM->hm.s.fNestedPaging)
1850 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1851 else
1852 {
1853 /*
1854 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1855 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1856 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1857 */
1858 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1859 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1860 }
1861
1862 if (pVM->hm.s.vmx.fVpid)
1863 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1864
1865 if (pVM->hm.s.vmx.fUnrestrictedGuest)
1866 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1867
1868 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1869 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1870 * done dynamically. */
1871 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1872 {
1873 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1874 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1875 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1876 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1877 AssertRCReturn(rc, rc);
1878 }
1879
1880 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1881 {
1882 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1883 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1884 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1885 }
1886
1887 if ((val & zap) != val)
1888 {
1889 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1890 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1891 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1892 }
1893
1894 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1895 AssertRCReturn(rc, rc);
1896
1897 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1898 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1899 }
1900 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1901 {
1902 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1903 "available\n"));
1904 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1905 }
1906
1907 return VINF_SUCCESS;
1908}
1909
1910
1911/**
1912 * Sets up miscellaneous (everything other than Pin & Processor-based
1913 * VM-execution) control fields in the VMCS.
1914 *
1915 * @returns VBox status code.
1916 * @param pVM Pointer to the VM.
1917 * @param pVCpu Pointer to the VMCPU.
1918 */
1919static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1920{
1921 AssertPtr(pVM);
1922 AssertPtr(pVCpu);
1923
1924 int rc = VERR_GENERAL_FAILURE;
1925
1926 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1927#if 0
1928 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
1929 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1930 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1931
1932 /*
1933 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1934 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1935 * We thus use the exception bitmap to control it rather than use both.
1936 */
1937 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1938 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1939
1940 /** @todo Explore possibility of using IO-bitmaps. */
1941 /* All IO & IOIO instructions cause VM-exits. */
1942 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1943 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1944
1945 /* Initialize the MSR-bitmap area. */
1946 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1947 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1948 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1949#endif
1950
1951#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1952 /* Setup MSR autoloading/storing. */
1953 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1954 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1955 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1956 AssertRCReturn(rc, rc);
1957 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1958 AssertRCReturn(rc, rc);
1959
1960 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1961 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1962 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1963 AssertRCReturn(rc, rc);
1964#endif
1965
1966 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1967 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1968 AssertRCReturn(rc, rc);
1969
1970 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1971#if 0
1972 /* Setup debug controls */
1973 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1974 AssertRCReturn(rc, rc);
1975 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1976 AssertRCReturn(rc, rc);
1977#endif
1978
1979 return rc;
1980}
1981
1982
1983/**
1984 * Sets up the initial exception bitmap in the VMCS based on static conditions
1985 * (i.e. conditions that cannot ever change after starting the VM).
1986 *
1987 * @returns VBox status code.
1988 * @param pVM Pointer to the VM.
1989 * @param pVCpu Pointer to the VMCPU.
1990 */
1991static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1992{
1993 AssertPtr(pVM);
1994 AssertPtr(pVCpu);
1995
1996 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1997
1998 uint32_t u32XcptBitmap = 0;
1999
2000 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2001 if (!pVM->hm.s.fNestedPaging)
2002 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2003
2004 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2005 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2006 AssertRCReturn(rc, rc);
2007 return rc;
2008}
2009
2010
2011/**
2012 * Sets up the initial guest-state mask. The guest-state mask is consulted
2013 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2014 * for the nested virtualization case (as it would cause a VM-exit).
2015 *
2016 * @param pVCpu Pointer to the VMCPU.
2017 */
2018static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2019{
2020 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2021 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2022 return VINF_SUCCESS;
2023}
2024
2025
2026/**
2027 * Does per-VM VT-x initialization.
2028 *
2029 * @returns VBox status code.
2030 * @param pVM Pointer to the VM.
2031 */
2032VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2033{
2034 LogFlowFunc(("pVM=%p\n", pVM));
2035
2036 int rc = hmR0VmxStructsAlloc(pVM);
2037 if (RT_FAILURE(rc))
2038 {
2039 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2040 return rc;
2041 }
2042
2043 return VINF_SUCCESS;
2044}
2045
2046
2047/**
2048 * Does per-VM VT-x termination.
2049 *
2050 * @returns VBox status code.
2051 * @param pVM Pointer to the VM.
2052 */
2053VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2054{
2055 LogFlowFunc(("pVM=%p\n", pVM));
2056
2057#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2058 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2059 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2060#endif
2061 hmR0VmxStructsFree(pVM);
2062 return VINF_SUCCESS;
2063}
2064
2065
2066/**
2067 * Sets up the VM for execution under VT-x.
2068 * This function is only called once per-VM during initialization.
2069 *
2070 * @returns VBox status code.
2071 * @param pVM Pointer to the VM.
2072 */
2073VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2074{
2075 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2076 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2077
2078 LogFlowFunc(("pVM=%p\n", pVM));
2079
2080 /*
2081 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2082 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2083 */
2084 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2085 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2086 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2087 || !pVM->hm.s.vmx.pRealModeTSS))
2088 {
2089 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2090 return VERR_INTERNAL_ERROR;
2091 }
2092
2093#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2094 /*
2095 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2096 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2097 */
2098 if ( pVM->hm.s.fAllow64BitGuests
2099 && !HMVMX_IS_64BIT_HOST_MODE())
2100 {
2101 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2102 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2103 }
2104#endif
2105
2106 /* Initialize these always, see hmR3InitFinalizeR0().*/
2107 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2108 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2109
2110 /* Setup the tagged-TLB flush handlers. */
2111 int rc = hmR0VmxSetupTaggedTlb(pVM);
2112 if (RT_FAILURE(rc))
2113 {
2114 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2115 return rc;
2116 }
2117
2118 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2119 {
2120 PVMCPU pVCpu = &pVM->aCpus[i];
2121 AssertPtr(pVCpu);
2122 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2123
2124 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2125 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2126
2127 /* Set revision dword at the beginning of the VMCS structure. */
2128 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2129
2130 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2131 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2132 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2133 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2134
2135 /* Load this VMCS as the current VMCS. */
2136 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2137 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2138 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2139
2140 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2141 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2142 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2143
2144 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2145 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2146 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2147
2148 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2149 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2150 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2151
2152 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2153 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2154 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2155
2156 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2157 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2158 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2159
2160#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2161 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2162 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2163 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2164#endif
2165
2166 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2167 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2168 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2169 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2170
2171 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2172
2173 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2174 }
2175
2176 return VINF_SUCCESS;
2177}
2178
2179
2180/**
2181 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2182 * the VMCS.
2183 *
2184 * @returns VBox status code.
2185 * @param pVM Pointer to the VM.
2186 * @param pVCpu Pointer to the VMCPU.
2187 */
2188DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2189{
2190 RTCCUINTREG uReg = ASMGetCR0();
2191 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2192 AssertRCReturn(rc, rc);
2193
2194#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2195 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2196 if (HMVMX_IS_64BIT_HOST_MODE())
2197 {
2198 uint64_t uRegCR3 = HMR0Get64bitCR3();
2199 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2200 }
2201 else
2202#endif
2203 {
2204 uReg = ASMGetCR3();
2205 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2206 }
2207 AssertRCReturn(rc, rc);
2208
2209 uReg = ASMGetCR4();
2210 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2211 AssertRCReturn(rc, rc);
2212 return rc;
2213}
2214
2215
2216/**
2217 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2218 * the host-state area in the VMCS.
2219 *
2220 * @returns VBox status code.
2221 * @param pVM Pointer to the VM.
2222 * @param pVCpu Pointer to the VMCPU.
2223 */
2224DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2225{
2226 int rc = VERR_INTERNAL_ERROR_5;
2227
2228 /*
2229 * Host DS, ES, FS and GS segment registers.
2230 */
2231#if HC_ARCH_BITS == 64
2232 RTSEL uSelDS = ASMGetDS();
2233 RTSEL uSelES = ASMGetES();
2234 RTSEL uSelFS = ASMGetFS();
2235 RTSEL uSelGS = ASMGetGS();
2236#else
2237 RTSEL uSelDS = 0;
2238 RTSEL uSelES = 0;
2239 RTSEL uSelFS = 0;
2240 RTSEL uSelGS = 0;
2241#endif
2242
2243 /* Recalculate which host-state bits need to be manually restored. */
2244 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2245
2246 /*
2247 * Host CS and SS segment registers.
2248 */
2249#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2250 RTSEL uSelCS;
2251 RTSEL uSelSS;
2252 if (HMVMX_IS_64BIT_HOST_MODE())
2253 {
2254 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2255 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2256 }
2257 else
2258 {
2259 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2260 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2261 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2262 }
2263#else
2264 RTSEL uSelCS = ASMGetCS();
2265 RTSEL uSelSS = ASMGetSS();
2266#endif
2267
2268 /*
2269 * Host TR segment register.
2270 */
2271 RTSEL uSelTR = ASMGetTR();
2272
2273#if HC_ARCH_BITS == 64
2274 /*
2275 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2276 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2277 */
2278 if (uSelDS & (X86_SEL_RPL | X86_SEL_LDT))
2279 {
2280 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_DS;
2281 pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2282 uSelDS = 0;
2283 }
2284 if (uSelES & (X86_SEL_RPL | X86_SEL_LDT))
2285 {
2286 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_ES;
2287 pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2288 uSelES = 0;
2289 }
2290 if (uSelFS & (X86_SEL_RPL | X86_SEL_LDT))
2291 {
2292 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_FS;
2293 pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2294 uSelFS = 0;
2295 }
2296 if (uSelGS & (X86_SEL_RPL | X86_SEL_LDT))
2297 {
2298 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_GS;
2299 pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2300 uSelGS = 0;
2301 }
2302#endif
2303
2304 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2305 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2306 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2307 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2308 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2309 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2310 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2311 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2312 Assert(uSelCS);
2313 Assert(uSelTR);
2314
2315 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2316#if 0
2317 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2318 Assert(uSelSS != 0);
2319#endif
2320
2321 /* Write these host selector fields into the host-state area in the VMCS. */
2322 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2323 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2324#if HC_ARCH_BITS == 64
2325 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2326 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2327 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2328 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2329#endif
2330 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2331
2332 /*
2333 * Host GDTR and IDTR.
2334 */
2335 RTGDTR Gdtr;
2336 RT_ZERO(Gdtr);
2337#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2338 if (HMVMX_IS_64BIT_HOST_MODE())
2339 {
2340 X86XDTR64 Gdtr64;
2341 X86XDTR64 Idtr64;
2342 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2343 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2344 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2345
2346 Gdtr.cbGdt = Gdtr64.cb;
2347 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2348 }
2349 else
2350#endif
2351 {
2352 RTIDTR Idtr;
2353 ASMGetGDTR(&Gdtr);
2354 ASMGetIDTR(&Idtr);
2355 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2356 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2357
2358#if HC_ARCH_BITS == 64
2359 /*
2360 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2361 * maximum limit (0xffff) on every VM-exit.
2362 */
2363 if (Gdtr.cbGdt != 0xffff)
2364 {
2365 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2366 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2367 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2368 }
2369
2370 /*
2371 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2372 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2373 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2374 */
2375 if (Idtr.cbIdt < 0x0fff)
2376 {
2377 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2378 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2379 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2380 }
2381#endif
2382 }
2383
2384 /*
2385 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2386 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2387 */
2388 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2389 {
2390 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2391 return VERR_VMX_INVALID_HOST_STATE;
2392 }
2393
2394 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2395#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2396 if (HMVMX_IS_64BIT_HOST_MODE())
2397 {
2398 /* We need the 64-bit TR base for hybrid darwin. */
2399 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2400 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2401 }
2402 else
2403#endif
2404 {
2405 uintptr_t uTRBase;
2406#if HC_ARCH_BITS == 64
2407 uTRBase = X86DESC64_BASE(pDesc);
2408
2409 /*
2410 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2411 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2412 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2413 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2414 *
2415 * [1] See Intel spec. 3.5 "System Descriptor Types".
2416 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2417 */
2418 Assert(pDesc->System.u4Type == 11);
2419 if ( pDesc->System.u16LimitLow != 0x67
2420 || pDesc->System.u4LimitHigh)
2421 {
2422 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2423 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2424
2425 /* Store the GDTR here as we need it while restoring TR. */
2426 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2427 }
2428#else
2429 uTRBase = X86DESC_BASE(pDesc);
2430#endif
2431 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2432 }
2433 AssertRCReturn(rc, rc);
2434
2435 /*
2436 * Host FS base and GS base.
2437 */
2438#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2439 if (HMVMX_IS_64BIT_HOST_MODE())
2440 {
2441 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2442 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2443 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2444 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2445
2446# if HC_ARCH_BITS == 64
2447 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2448 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2449 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2450 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2451 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2452# endif
2453 }
2454#endif
2455 return rc;
2456}
2457
2458
2459/**
2460 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2461 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2462 * the host after every successful VM exit.
2463 *
2464 * @returns VBox status code.
2465 * @param pVM Pointer to the VM.
2466 * @param pVCpu Pointer to the VMCPU.
2467 */
2468DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2469{
2470 AssertPtr(pVCpu);
2471 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2472
2473 int rc = VINF_SUCCESS;
2474#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2475 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2476 uint32_t cHostMsrs = 0;
2477 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2478
2479 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2480 {
2481 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2482
2483# if HC_ARCH_BITS == 64
2484 /* Paranoia. 64-bit code requires these bits to be set always. */
2485 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2486
2487 /*
2488 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2489 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2490 * some reason (e.g. allow transparent reads) we would activate the code below.
2491 */
2492# if 0
2493 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2494 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2495 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2496 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2497 if (CPUMIsGuestInLongMode(pVCpu))
2498 {
2499 uint64_t u64GuestEfer;
2500 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2501 AssertRC(rc);
2502
2503 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2504 {
2505 pHostMsr->u32Msr = MSR_K6_EFER;
2506 pHostMsr->u32Reserved = 0;
2507 pHostMsr->u64Value = u64HostEfer;
2508 pHostMsr++; cHostMsrs++;
2509 }
2510 }
2511# endif
2512# else /* HC_ARCH_BITS != 64 */
2513 pHostMsr->u32Msr = MSR_K6_EFER;
2514 pHostMsr->u32Reserved = 0;
2515# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2516 if (CPUMIsGuestInLongMode(pVCpu))
2517 {
2518 /* Must match the EFER value in our 64 bits switcher. */
2519 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2520 }
2521 else
2522# endif
2523 pHostMsr->u64Value = u64HostEfer;
2524 pHostMsr++; cHostMsrs++;
2525# endif /* HC_ARCH_BITS == 64 */
2526 }
2527
2528# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2529 if (HMVMX_IS_64BIT_HOST_MODE())
2530 {
2531 pHostMsr->u32Msr = MSR_K6_STAR;
2532 pHostMsr->u32Reserved = 0;
2533 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2534 pHostMsr++; cHostMsrs++;
2535 pHostMsr->u32Msr = MSR_K8_LSTAR;
2536 pHostMsr->u32Reserved = 0;
2537 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2538 pHostMsr++; cHostMsrs++;
2539 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2540 pHostMsr->u32Reserved = 0;
2541 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2542 pHostMsr++; cHostMsrs++;
2543 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2544 pHostMsr->u32Reserved = 0;
2545 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2546 pHostMsr++; cHostMsrs++;
2547 }
2548# endif
2549
2550 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2551 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2552 {
2553 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2554 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2555 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2556 }
2557
2558 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2559#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2560
2561 /*
2562 * Host Sysenter MSRs.
2563 */
2564 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2565 AssertRCReturn(rc, rc);
2566#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2567 if (HMVMX_IS_64BIT_HOST_MODE())
2568 {
2569 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2570 AssertRCReturn(rc, rc);
2571 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2572 }
2573 else
2574 {
2575 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2576 AssertRCReturn(rc, rc);
2577 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2578 }
2579#elif HC_ARCH_BITS == 32
2580 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2581 AssertRCReturn(rc, rc);
2582 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2583#else
2584 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2585 AssertRCReturn(rc, rc);
2586 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2587#endif
2588 AssertRCReturn(rc, rc);
2589
2590 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2591 * hmR0VmxSetupExitCtls() !! */
2592 return rc;
2593}
2594
2595
2596/**
2597 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2598 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2599 * controls".
2600 *
2601 * @returns VBox status code.
2602 * @param pVCpu Pointer to the VMCPU.
2603 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2604 * out-of-sync. Make sure to update the required fields
2605 * before using them.
2606 *
2607 * @remarks No-long-jump zone!!!
2608 */
2609DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2610{
2611 int rc = VINF_SUCCESS;
2612 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2613 {
2614 PVM pVM = pVCpu->CTX_SUFF(pVM);
2615 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2616 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2617
2618 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2619 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2620
2621 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2622 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2623 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2624 else
2625 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2626
2627 /*
2628 * The following should -not- be set (since we're not in SMM mode):
2629 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2630 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2631 */
2632
2633 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2634 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2635 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2636
2637 if ((val & zap) != val)
2638 {
2639 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2640 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2641 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2642 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2643 }
2644
2645 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2646 AssertRCReturn(rc, rc);
2647
2648 /* Update VCPU with the currently set VM-exit controls. */
2649 pVCpu->hm.s.vmx.u32EntryCtls = val;
2650 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2651 }
2652 return rc;
2653}
2654
2655
2656/**
2657 * Sets up the VM-exit controls in the VMCS.
2658 *
2659 * @returns VBox status code.
2660 * @param pVM Pointer to the VM.
2661 * @param pVCpu Pointer to the VMCPU.
2662 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2663 * out-of-sync. Make sure to update the required fields
2664 * before using them.
2665 *
2666 * @remarks requires EFER.
2667 */
2668DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2669{
2670 int rc = VINF_SUCCESS;
2671 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2672 {
2673 PVM pVM = pVCpu->CTX_SUFF(pVM);
2674 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2675 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2676
2677 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2678 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2679
2680 /*
2681 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2682 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2683 */
2684#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2685 if (HMVMX_IS_64BIT_HOST_MODE())
2686 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2687 else
2688 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2689#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2690 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2691 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2692 else
2693 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2694#endif
2695
2696 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2697 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2698
2699 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2700 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2701 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2702 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2703 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2704
2705 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2706 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2707
2708 if ((val & zap) != val)
2709 {
2710 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2711 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2712 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2713 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2714 }
2715
2716 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2717 AssertRCReturn(rc, rc);
2718
2719 /* Update VCPU with the currently set VM-exit controls. */
2720 pVCpu->hm.s.vmx.u32ExitCtls = val;
2721 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2722 }
2723 return rc;
2724}
2725
2726
2727/**
2728 * Loads the guest APIC and related state.
2729 *
2730 * @returns VBox status code.
2731 * @param pVM Pointer to the VM.
2732 * @param pVCpu Pointer to the VMCPU.
2733 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2734 * out-of-sync. Make sure to update the required fields
2735 * before using them.
2736 */
2737DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2738{
2739 int rc = VINF_SUCCESS;
2740 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2741 {
2742 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2743 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2744 {
2745 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2746
2747 bool fPendingIntr = false;
2748 uint8_t u8Tpr = 0;
2749 uint8_t u8PendingIntr = 0;
2750 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2751 AssertRCReturn(rc, rc);
2752
2753 /*
2754 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2755 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2756 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2757 * the interrupt when we VM-exit for other reasons.
2758 */
2759 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2760 uint32_t u32TprThreshold = 0;
2761 if (fPendingIntr)
2762 {
2763 /* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2764 const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2765 const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2766 if (u8PendingPriority <= u8TprPriority)
2767 u32TprThreshold = u8PendingPriority;
2768 else
2769 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2770 }
2771 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2772
2773 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2774 AssertRCReturn(rc, rc);
2775 }
2776
2777 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2778 }
2779 return rc;
2780}
2781
2782
2783/**
2784 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2785 *
2786 * @returns Guest's interruptibility-state.
2787 * @param pVCpu Pointer to the VMCPU.
2788 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2789 * out-of-sync. Make sure to update the required fields
2790 * before using them.
2791 *
2792 * @remarks No-long-jump zone!!!
2793 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2794 */
2795DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2796{
2797 /*
2798 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2799 * inhibit interrupts or clear any existing interrupt-inhibition.
2800 */
2801 uint32_t uIntrState = 0;
2802 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2803 {
2804 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2805 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
2806 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2807 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2808 {
2809 /*
2810 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2811 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2812 */
2813 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2814 }
2815 else if (pMixedCtx->eflags.Bits.u1IF)
2816 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2817 else
2818 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2819 }
2820 return uIntrState;
2821}
2822
2823
2824/**
2825 * Loads the guest's interruptibility-state into the guest-state area in the
2826 * VMCS.
2827 *
2828 * @returns VBox status code.
2829 * @param pVCpu Pointer to the VMCPU.
2830 * @param uIntrState The interruptibility-state to set.
2831 */
2832static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2833{
2834 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2835 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2836 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2837 AssertRCReturn(rc, rc);
2838 return rc;
2839}
2840
2841
2842/**
2843 * Loads the guest's RIP into the guest-state area in the VMCS.
2844 *
2845 * @returns VBox status code.
2846 * @param pVCpu Pointer to the VMCPU.
2847 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2848 * out-of-sync. Make sure to update the required fields
2849 * before using them.
2850 *
2851 * @remarks No-long-jump zone!!!
2852 */
2853static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2854{
2855 int rc = VINF_SUCCESS;
2856 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2857 {
2858 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2859 AssertRCReturn(rc, rc);
2860 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2861 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#x\n", pMixedCtx->rip, pVCpu->hm.s.fContextUseFlags));
2862 }
2863 return rc;
2864}
2865
2866
2867/**
2868 * Loads the guest's RSP into the guest-state area in the VMCS.
2869 *
2870 * @returns VBox status code.
2871 * @param pVCpu Pointer to the VMCPU.
2872 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2873 * out-of-sync. Make sure to update the required fields
2874 * before using them.
2875 *
2876 * @remarks No-long-jump zone!!!
2877 */
2878static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2879{
2880 int rc = VINF_SUCCESS;
2881 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2882 {
2883 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2884 AssertRCReturn(rc, rc);
2885 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2886 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2887 }
2888 return rc;
2889}
2890
2891
2892/**
2893 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
2894 *
2895 * @returns VBox status code.
2896 * @param pVCpu Pointer to the VMCPU.
2897 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2898 * out-of-sync. Make sure to update the required fields
2899 * before using them.
2900 *
2901 * @remarks No-long-jump zone!!!
2902 */
2903static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2904{
2905 int rc = VINF_SUCCESS;
2906 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2907 {
2908 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2909 Let us assert it as such and use 32-bit VMWRITE. */
2910 Assert(!(pMixedCtx->rflags.u64 >> 32));
2911 X86EFLAGS Eflags = pMixedCtx->eflags;
2912 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2913 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2914
2915 /*
2916 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2917 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2918 */
2919 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2920 {
2921 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2922 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2923 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2924 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2925 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2926 }
2927
2928 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2929 AssertRCReturn(rc, rc);
2930
2931 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2932 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2933 }
2934 return rc;
2935}
2936
2937
2938/**
2939 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2940 *
2941 * @returns VBox status code.
2942 * @param pVCpu Pointer to the VMCPU.
2943 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2944 * out-of-sync. Make sure to update the required fields
2945 * before using them.
2946 *
2947 * @remarks No-long-jump zone!!!
2948 */
2949DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2950{
2951 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2952 AssertRCReturn(rc, rc);
2953 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2954 AssertRCReturn(rc, rc);
2955 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2956 AssertRCReturn(rc, rc);
2957 return rc;
2958}
2959
2960
2961/**
2962 * Loads the guest CR0 control register into the guest-state area in the VMCS.
2963 * CR0 is partially shared with the host and we have to consider the FPU bits.
2964 *
2965 * @returns VBox status code.
2966 * @param pVM Pointer to the VM.
2967 * @param pVCpu Pointer to the VMCPU.
2968 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2969 * out-of-sync. Make sure to update the required fields
2970 * before using them.
2971 *
2972 * @remarks No-long-jump zone!!!
2973 */
2974static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2975{
2976 /*
2977 * Guest CR0.
2978 * Guest FPU.
2979 */
2980 int rc = VINF_SUCCESS;
2981 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2982 {
2983 Assert(!(pMixedCtx->cr0 >> 32));
2984 uint32_t u32GuestCR0 = pMixedCtx->cr0;
2985 PVM pVM = pVCpu->CTX_SUFF(pVM);
2986
2987 /* The guest's view (read access) of its CR0 is unblemished. */
2988 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2989 AssertRCReturn(rc, rc);
2990 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2991
2992 /* Setup VT-x's view of the guest CR0. */
2993 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2994 if (pVM->hm.s.fNestedPaging)
2995 {
2996 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
2997 {
2998 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2999 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3000 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3001 }
3002 else
3003 {
3004 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3005 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3006 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3007 }
3008
3009 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3010 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3011 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3012
3013 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3014 AssertRCReturn(rc, rc);
3015 }
3016 else
3017 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3018
3019 /*
3020 * Guest FPU bits.
3021 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3022 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3023 */
3024 u32GuestCR0 |= X86_CR0_NE;
3025 bool fInterceptNM = false;
3026 if (CPUMIsGuestFPUStateActive(pVCpu))
3027 {
3028 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3029 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3030 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3031 }
3032 else
3033 {
3034 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3035 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3036 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3037 }
3038
3039 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3040 bool fInterceptMF = false;
3041 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3042 fInterceptMF = true;
3043
3044 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3045 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3046 {
3047 Assert(PDMVmmDevHeapIsEnabled(pVM));
3048 Assert(pVM->hm.s.vmx.pRealModeTSS);
3049 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3050 fInterceptNM = true;
3051 fInterceptMF = true;
3052 }
3053 else
3054 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3055
3056 if (fInterceptNM)
3057 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3058 else
3059 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3060
3061 if (fInterceptMF)
3062 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3063 else
3064 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3065
3066 /* Additional intercepts for debugging, define these yourself explicitly. */
3067#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3068 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3069 | RT_BIT(X86_XCPT_BP)
3070 | RT_BIT(X86_XCPT_DB)
3071 | RT_BIT(X86_XCPT_DE)
3072 | RT_BIT(X86_XCPT_NM)
3073 | RT_BIT(X86_XCPT_UD)
3074 | RT_BIT(X86_XCPT_NP)
3075 | RT_BIT(X86_XCPT_SS)
3076 | RT_BIT(X86_XCPT_GP)
3077 | RT_BIT(X86_XCPT_PF)
3078 | RT_BIT(X86_XCPT_MF)
3079 ;
3080#elif defined(HMVMX_ALWAYS_TRAP_PF)
3081 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3082#endif
3083
3084 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3085
3086 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3087 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3088 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3089 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3090 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3091 else
3092 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3093
3094 u32GuestCR0 |= uSetCR0;
3095 u32GuestCR0 &= uZapCR0;
3096 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3097
3098 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3099 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3100 AssertRCReturn(rc, rc);
3101 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3102 AssertRCReturn(rc, rc);
3103 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3104
3105 /*
3106 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3107 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3108 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3109 */
3110 uint32_t u32CR0Mask = 0;
3111 u32CR0Mask = X86_CR0_PE
3112 | X86_CR0_NE
3113 | X86_CR0_WP
3114 | X86_CR0_PG
3115 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3116 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3117 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3118
3119 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3120 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3121 * and @bugref{6944}. */
3122#if 0
3123 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3124 u32CR0Mask &= ~X86_CR0_PE;
3125#endif
3126 if (pVM->hm.s.fNestedPaging)
3127 u32CR0Mask &= ~X86_CR0_WP;
3128
3129 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3130 if (fInterceptNM)
3131 u32CR0Mask |= (X86_CR0_TS | X86_CR0_MP);
3132 else
3133 u32CR0Mask &= ~(X86_CR0_TS | X86_CR0_MP);
3134
3135 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3136 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3137 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3138 AssertRCReturn(rc, rc);
3139
3140 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3141 }
3142 return rc;
3143}
3144
3145
3146/**
3147 * Loads the guest control registers (CR3, CR4) into the guest-state area
3148 * in the VMCS.
3149 *
3150 * @returns VBox status code.
3151 * @param pVM Pointer to the VM.
3152 * @param pVCpu Pointer to the VMCPU.
3153 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3154 * out-of-sync. Make sure to update the required fields
3155 * before using them.
3156 *
3157 * @remarks No-long-jump zone!!!
3158 */
3159static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3160{
3161 int rc = VINF_SUCCESS;
3162 PVM pVM = pVCpu->CTX_SUFF(pVM);
3163
3164 /*
3165 * Guest CR2.
3166 * It's always loaded in the assembler code. Nothing to do here.
3167 */
3168
3169 /*
3170 * Guest CR3.
3171 */
3172 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3173 {
3174 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3175 if (pVM->hm.s.fNestedPaging)
3176 {
3177 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3178
3179 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3180 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3181 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3182 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3183
3184 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3185 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3186 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3187
3188 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3189 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3190 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3191 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3192
3193 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3194 AssertRCReturn(rc, rc);
3195 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3196
3197 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3198 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3199 {
3200 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3201 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3202 {
3203 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3204 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3205 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3206 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3207 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3208 }
3209
3210 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3211 have Unrestricted Execution to handle the guest when it's not using paging. */
3212 GCPhysGuestCR3 = pMixedCtx->cr3;
3213 }
3214 else
3215 {
3216 /*
3217 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3218 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3219 * EPT takes care of translating it to host-physical addresses.
3220 */
3221 RTGCPHYS GCPhys;
3222 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3223 Assert(PDMVmmDevHeapIsEnabled(pVM));
3224
3225 /* We obtain it here every time as the guest could have relocated this PCI region. */
3226 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3227 AssertRCReturn(rc, rc);
3228
3229 GCPhysGuestCR3 = GCPhys;
3230 }
3231
3232 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3233 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3234 }
3235 else
3236 {
3237 /* Non-nested paging case, just use the hypervisor's CR3. */
3238 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3239
3240 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3241 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3242 }
3243 AssertRCReturn(rc, rc);
3244
3245 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3246 }
3247
3248 /*
3249 * Guest CR4.
3250 */
3251 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3252 {
3253 Assert(!(pMixedCtx->cr4 >> 32));
3254 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3255
3256 /* The guest's view of its CR4 is unblemished. */
3257 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3258 AssertRCReturn(rc, rc);
3259 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3260
3261 /* Setup VT-x's view of the guest CR4. */
3262 /*
3263 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3264 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3265 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3266 */
3267 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3268 {
3269 Assert(pVM->hm.s.vmx.pRealModeTSS);
3270 Assert(PDMVmmDevHeapIsEnabled(pVM));
3271 u32GuestCR4 &= ~X86_CR4_VME;
3272 }
3273
3274 if (pVM->hm.s.fNestedPaging)
3275 {
3276 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3277 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3278 {
3279 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3280 u32GuestCR4 |= X86_CR4_PSE;
3281 /* Our identity mapping is a 32 bits page directory. */
3282 u32GuestCR4 &= ~X86_CR4_PAE;
3283 }
3284 /* else use guest CR4.*/
3285 }
3286 else
3287 {
3288 /*
3289 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3290 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3291 */
3292 switch (pVCpu->hm.s.enmShadowMode)
3293 {
3294 case PGMMODE_REAL: /* Real-mode. */
3295 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3296 case PGMMODE_32_BIT: /* 32-bit paging. */
3297 {
3298 u32GuestCR4 &= ~X86_CR4_PAE;
3299 break;
3300 }
3301
3302 case PGMMODE_PAE: /* PAE paging. */
3303 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3304 {
3305 u32GuestCR4 |= X86_CR4_PAE;
3306 break;
3307 }
3308
3309 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3310 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3311#ifdef VBOX_ENABLE_64_BITS_GUESTS
3312 break;
3313#endif
3314 default:
3315 AssertFailed();
3316 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3317 }
3318 }
3319
3320 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3321 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3322 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3323 u32GuestCR4 |= uSetCR4;
3324 u32GuestCR4 &= uZapCR4;
3325
3326 /* Write VT-x's view of the guest CR4 into the VMCS. */
3327 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3328 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3329 AssertRCReturn(rc, rc);
3330
3331 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3332 uint32_t u32CR4Mask = 0;
3333 u32CR4Mask = X86_CR4_VME
3334 | X86_CR4_PAE
3335 | X86_CR4_PGE
3336 | X86_CR4_PSE
3337 | X86_CR4_VMXE;
3338 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3339 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3340 AssertRCReturn(rc, rc);
3341
3342 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3343 }
3344 return rc;
3345}
3346
3347
3348/**
3349 * Loads the guest debug registers into the guest-state area in the VMCS.
3350 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3351 *
3352 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3353 *
3354 * @returns VBox status code.
3355 * @param pVCpu Pointer to the VMCPU.
3356 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3357 * out-of-sync. Make sure to update the required fields
3358 * before using them.
3359 *
3360 * @remarks No-long-jump zone!!!
3361 */
3362static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3363{
3364 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3365 return VINF_SUCCESS;
3366
3367#ifdef VBOX_STRICT
3368 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3369 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3370 {
3371 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3372 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3373 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3374 }
3375#endif
3376
3377 int rc;
3378 PVM pVM = pVCpu->CTX_SUFF(pVM);
3379 bool fInterceptDB = false;
3380 bool fInterceptMovDRx = false;
3381 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3382 {
3383 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3384 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3385 {
3386 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3387 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3388 AssertRCReturn(rc, rc);
3389 Assert(fInterceptDB == false);
3390 }
3391 else
3392 {
3393 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3394 pVCpu->hm.s.fClearTrapFlag = true;
3395 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RFLAGS;
3396 fInterceptDB = true;
3397 }
3398 }
3399
3400 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3401 {
3402 /*
3403 * Use the combined guest and host DRx values found in the hypervisor
3404 * register set because the debugger has breakpoints active or someone
3405 * is single stepping on the host side without a monitor trap flag.
3406 *
3407 * Note! DBGF expects a clean DR6 state before executing guest code.
3408 */
3409 if (!CPUMIsHyperDebugStateActive(pVCpu))
3410 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3411 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3412 Assert(CPUMIsHyperDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3413
3414 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3415 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3416 AssertRCReturn(rc, rc);
3417
3418 fInterceptDB = true;
3419 fInterceptMovDRx = true;
3420 }
3421 else
3422 {
3423 /*
3424 * If the guest has enabled debug registers, we need to load them prior to
3425 * executing guest code so they'll trigger at the right time.
3426 */
3427 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3428 {
3429 if (!CPUMIsGuestDebugStateActive(pVCpu))
3430 {
3431 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3432 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3433 }
3434 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3435 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3436 }
3437 /*
3438 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3439 * must intercept #DB in order to maintain a correct DR6 guest value.
3440 */
3441 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3442 {
3443 fInterceptMovDRx = true;
3444 fInterceptDB = true;
3445 }
3446
3447 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3448 AssertRCReturn(rc, rc);
3449 }
3450
3451 /*
3452 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3453 */
3454 if (fInterceptDB)
3455 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3456 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3457 {
3458#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3459 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3460#endif
3461 }
3462 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3463 AssertRCReturn(rc, rc);
3464
3465 /*
3466 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3467 */
3468 if (fInterceptMovDRx)
3469 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3470 else
3471 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3472 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3473 AssertRCReturn(rc, rc);
3474
3475 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3476 return VINF_SUCCESS;
3477}
3478
3479
3480#ifdef VBOX_STRICT
3481/**
3482 * Strict function to validate segment registers.
3483 *
3484 * @remarks ASSUMES CR0 is up to date.
3485 */
3486static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3487{
3488 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3489 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3490 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3491 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3492 && ( !CPUMIsGuestInRealModeEx(pCtx)
3493 && !CPUMIsGuestInV86ModeEx(pCtx)))
3494 {
3495 /* Protected mode checks */
3496 /* CS */
3497 Assert(pCtx->cs.Attr.n.u1Present);
3498 Assert(!(pCtx->cs.Attr.u & 0xf00));
3499 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3500 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3501 || !(pCtx->cs.Attr.n.u1Granularity));
3502 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3503 || (pCtx->cs.Attr.n.u1Granularity));
3504 /* CS cannot be loaded with NULL in protected mode. */
3505 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3506 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3507 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3508 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3509 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3510 else
3511 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3512 /* SS */
3513 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3514 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3515 if ( !(pCtx->cr0 & X86_CR0_PE)
3516 || pCtx->cs.Attr.n.u4Type == 3)
3517 {
3518 Assert(!pCtx->ss.Attr.n.u2Dpl);
3519 }
3520 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3521 {
3522 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3523 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3524 Assert(pCtx->ss.Attr.n.u1Present);
3525 Assert(!(pCtx->ss.Attr.u & 0xf00));
3526 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3527 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3528 || !(pCtx->ss.Attr.n.u1Granularity));
3529 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3530 || (pCtx->ss.Attr.n.u1Granularity));
3531 }
3532 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3533 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3534 {
3535 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3536 Assert(pCtx->ds.Attr.n.u1Present);
3537 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3538 Assert(!(pCtx->ds.Attr.u & 0xf00));
3539 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3540 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3541 || !(pCtx->ds.Attr.n.u1Granularity));
3542 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3543 || (pCtx->ds.Attr.n.u1Granularity));
3544 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3545 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3546 }
3547 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3548 {
3549 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3550 Assert(pCtx->es.Attr.n.u1Present);
3551 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3552 Assert(!(pCtx->es.Attr.u & 0xf00));
3553 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3554 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3555 || !(pCtx->es.Attr.n.u1Granularity));
3556 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3557 || (pCtx->es.Attr.n.u1Granularity));
3558 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3559 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3560 }
3561 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3562 {
3563 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3564 Assert(pCtx->fs.Attr.n.u1Present);
3565 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3566 Assert(!(pCtx->fs.Attr.u & 0xf00));
3567 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3568 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3569 || !(pCtx->fs.Attr.n.u1Granularity));
3570 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3571 || (pCtx->fs.Attr.n.u1Granularity));
3572 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3573 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3574 }
3575 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3576 {
3577 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3578 Assert(pCtx->gs.Attr.n.u1Present);
3579 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3580 Assert(!(pCtx->gs.Attr.u & 0xf00));
3581 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3582 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3583 || !(pCtx->gs.Attr.n.u1Granularity));
3584 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3585 || (pCtx->gs.Attr.n.u1Granularity));
3586 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3587 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3588 }
3589 /* 64-bit capable CPUs. */
3590# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3591 Assert(!(pCtx->cs.u64Base >> 32));
3592 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3593 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3594 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3595# endif
3596 }
3597 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3598 || ( CPUMIsGuestInRealModeEx(pCtx)
3599 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3600 {
3601 /* Real and v86 mode checks. */
3602 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3603 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3604 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3605 {
3606 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3607 }
3608 else
3609 {
3610 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3611 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3612 }
3613
3614 /* CS */
3615 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3616 Assert(pCtx->cs.u32Limit == 0xffff);
3617 Assert(u32CSAttr == 0xf3);
3618 /* SS */
3619 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3620 Assert(pCtx->ss.u32Limit == 0xffff);
3621 Assert(u32SSAttr == 0xf3);
3622 /* DS */
3623 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3624 Assert(pCtx->ds.u32Limit == 0xffff);
3625 Assert(u32DSAttr == 0xf3);
3626 /* ES */
3627 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3628 Assert(pCtx->es.u32Limit == 0xffff);
3629 Assert(u32ESAttr == 0xf3);
3630 /* FS */
3631 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3632 Assert(pCtx->fs.u32Limit == 0xffff);
3633 Assert(u32FSAttr == 0xf3);
3634 /* GS */
3635 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3636 Assert(pCtx->gs.u32Limit == 0xffff);
3637 Assert(u32GSAttr == 0xf3);
3638 /* 64-bit capable CPUs. */
3639# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3640 Assert(!(pCtx->cs.u64Base >> 32));
3641 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3642 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3643 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3644# endif
3645 }
3646}
3647#endif /* VBOX_STRICT */
3648
3649
3650/**
3651 * Writes a guest segment register into the guest-state area in the VMCS.
3652 *
3653 * @returns VBox status code.
3654 * @param pVCpu Pointer to the VMCPU.
3655 * @param idxSel Index of the selector in the VMCS.
3656 * @param idxLimit Index of the segment limit in the VMCS.
3657 * @param idxBase Index of the segment base in the VMCS.
3658 * @param idxAccess Index of the access rights of the segment in the VMCS.
3659 * @param pSelReg Pointer to the segment selector.
3660 * @param pCtx Pointer to the guest-CPU context.
3661 *
3662 * @remarks No-long-jump zone!!!
3663 */
3664static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3665 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3666{
3667 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3668 AssertRCReturn(rc, rc);
3669 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3670 AssertRCReturn(rc, rc);
3671 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3672 AssertRCReturn(rc, rc);
3673
3674 uint32_t u32Access = pSelReg->Attr.u;
3675 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3676 {
3677 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3678 u32Access = 0xf3;
3679 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3680 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3681 }
3682 else
3683 {
3684 /*
3685 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3686 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3687 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3688 * loaded in protected-mode have their attribute as 0.
3689 */
3690 if (!u32Access)
3691 u32Access = X86DESCATTR_UNUSABLE;
3692 }
3693
3694 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3695 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3696 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3697
3698 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3699 AssertRCReturn(rc, rc);
3700 return rc;
3701}
3702
3703
3704/**
3705 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3706 * into the guest-state area in the VMCS.
3707 *
3708 * @returns VBox status code.
3709 * @param pVM Pointer to the VM.
3710 * @param pVCPU Pointer to the VMCPU.
3711 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3712 * out-of-sync. Make sure to update the required fields
3713 * before using them.
3714 *
3715 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
3716 * @remarks No-long-jump zone!!!
3717 */
3718static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3719{
3720 int rc = VERR_INTERNAL_ERROR_5;
3721 PVM pVM = pVCpu->CTX_SUFF(pVM);
3722
3723 /*
3724 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
3725 */
3726 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3727 {
3728 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3729 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3730 {
3731 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3732 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3733 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3734 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3735 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3736 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3737 }
3738
3739#ifdef VBOX_WITH_REM
3740 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3741 {
3742 Assert(pVM->hm.s.vmx.pRealModeTSS);
3743 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3744 if ( pVCpu->hm.s.vmx.fWasInRealMode
3745 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3746 {
3747 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3748 in real-mode (e.g. OpenBSD 4.0) */
3749 REMFlushTBs(pVM);
3750 Log4(("Load: Switch to protected mode detected!\n"));
3751 pVCpu->hm.s.vmx.fWasInRealMode = false;
3752 }
3753 }
3754#endif
3755 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3756 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3757 AssertRCReturn(rc, rc);
3758 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3759 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3760 AssertRCReturn(rc, rc);
3761 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3762 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3763 AssertRCReturn(rc, rc);
3764 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3765 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3766 AssertRCReturn(rc, rc);
3767 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3768 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3769 AssertRCReturn(rc, rc);
3770 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3771 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3772 AssertRCReturn(rc, rc);
3773
3774 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3775 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3776#ifdef VBOX_STRICT
3777 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3778#endif
3779 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3780 }
3781
3782 /*
3783 * Guest TR.
3784 */
3785 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3786 {
3787 /*
3788 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3789 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3790 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3791 */
3792 uint16_t u16Sel = 0;
3793 uint32_t u32Limit = 0;
3794 uint64_t u64Base = 0;
3795 uint32_t u32AccessRights = 0;
3796
3797 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3798 {
3799 u16Sel = pMixedCtx->tr.Sel;
3800 u32Limit = pMixedCtx->tr.u32Limit;
3801 u64Base = pMixedCtx->tr.u64Base;
3802 u32AccessRights = pMixedCtx->tr.Attr.u;
3803 }
3804 else
3805 {
3806 Assert(pVM->hm.s.vmx.pRealModeTSS);
3807 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3808
3809 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3810 RTGCPHYS GCPhys;
3811 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3812 AssertRCReturn(rc, rc);
3813
3814 X86DESCATTR DescAttr;
3815 DescAttr.u = 0;
3816 DescAttr.n.u1Present = 1;
3817 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3818
3819 u16Sel = 0;
3820 u32Limit = HM_VTX_TSS_SIZE;
3821 u64Base = GCPhys; /* in real-mode phys = virt. */
3822 u32AccessRights = DescAttr.u;
3823 }
3824
3825 /* Validate. */
3826 Assert(!(u16Sel & RT_BIT(2)));
3827 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3828 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3829 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3830 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3831 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3832 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3833 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3834 Assert( (u32Limit & 0xfff) == 0xfff
3835 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3836 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3837 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3838
3839 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3840 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3841 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3842 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3843
3844 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3845 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3846 }
3847
3848 /*
3849 * Guest GDTR.
3850 */
3851 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3852 {
3853 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3854 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3855
3856 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3857 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3858 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3859 }
3860
3861 /*
3862 * Guest LDTR.
3863 */
3864 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3865 {
3866 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3867 uint32_t u32Access = 0;
3868 if (!pMixedCtx->ldtr.Attr.u)
3869 u32Access = X86DESCATTR_UNUSABLE;
3870 else
3871 u32Access = pMixedCtx->ldtr.Attr.u;
3872
3873 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3874 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3875 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3876 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3877
3878 /* Validate. */
3879 if (!(u32Access & X86DESCATTR_UNUSABLE))
3880 {
3881 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3882 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3883 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3884 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3885 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3886 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3887 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3888 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3889 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3890 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3891 }
3892
3893 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3894 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3895 }
3896
3897 /*
3898 * Guest IDTR.
3899 */
3900 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3901 {
3902 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3903 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3904
3905 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3906 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3907 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3908 }
3909
3910 return VINF_SUCCESS;
3911}
3912
3913
3914/**
3915 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3916 * areas. These MSRs will automatically be loaded to the host CPU on every
3917 * successful VM entry and stored from the host CPU on every successful VM exit.
3918 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
3919 *
3920 * @returns VBox status code.
3921 * @param pVCpu Pointer to the VMCPU.
3922 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3923 * out-of-sync. Make sure to update the required fields
3924 * before using them.
3925 *
3926 * @remarks No-long-jump zone!!!
3927 */
3928static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3929{
3930 AssertPtr(pVCpu);
3931 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3932
3933 /*
3934 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3935 */
3936 int rc = VINF_SUCCESS;
3937 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3938 {
3939#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3940 PVM pVM = pVCpu->CTX_SUFF(pVM);
3941 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3942 uint32_t cGuestMsrs = 0;
3943
3944 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3945 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3946 * when the guest really is in 64-bit mode. */
3947 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3948 if (fSupportsLongMode)
3949 {
3950 pGuestMsr->u32Msr = MSR_K8_LSTAR;
3951 pGuestMsr->u32Reserved = 0;
3952 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3953 pGuestMsr++; cGuestMsrs++;
3954 pGuestMsr->u32Msr = MSR_K6_STAR;
3955 pGuestMsr->u32Reserved = 0;
3956 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3957 pGuestMsr++; cGuestMsrs++;
3958 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
3959 pGuestMsr->u32Reserved = 0;
3960 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3961 pGuestMsr++; cGuestMsrs++;
3962 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
3963 pGuestMsr->u32Reserved = 0;
3964 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3965 pGuestMsr++; cGuestMsrs++;
3966 }
3967
3968 /*
3969 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3970 * load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3971 */
3972 if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3973 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3974 {
3975 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
3976 pGuestMsr->u32Reserved = 0;
3977 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3978 AssertRCReturn(rc, rc);
3979 pGuestMsr++; cGuestMsrs++;
3980 }
3981
3982 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3983 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
3984 {
3985 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3986 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
3987 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3988 }
3989
3990 /* Update the VCPU's copy of the guest MSR count. */
3991 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3992 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3993 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3994#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3995
3996 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3997 }
3998
3999 /*
4000 * Guest Sysenter MSRs.
4001 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4002 * VM-exits on WRMSRs for these MSRs.
4003 */
4004 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4005 {
4006 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4007 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
4008 }
4009 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4010 {
4011 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4012 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
4013 }
4014 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4015 {
4016 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4017 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
4018 }
4019
4020 return rc;
4021}
4022
4023
4024/**
4025 * Loads the guest activity state into the guest-state area in the VMCS.
4026 *
4027 * @returns VBox status code.
4028 * @param pVCpu Pointer to the VMCPU.
4029 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4030 * out-of-sync. Make sure to update the required fields
4031 * before using them.
4032 *
4033 * @remarks No-long-jump zone!!!
4034 */
4035static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4036{
4037 /** @todo See if we can make use of other states, e.g.
4038 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4039 int rc = VINF_SUCCESS;
4040 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
4041 {
4042 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4043 AssertRCReturn(rc, rc);
4044 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
4045 }
4046 return rc;
4047}
4048
4049
4050/**
4051 * Sets up the appropriate function to run guest code.
4052 *
4053 * @returns VBox status code.
4054 * @param pVCpu Pointer to the VMCPU.
4055 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4056 * out-of-sync. Make sure to update the required fields
4057 * before using them.
4058 *
4059 * @remarks No-long-jump zone!!!
4060 */
4061static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4062{
4063 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4064 {
4065#ifndef VBOX_ENABLE_64_BITS_GUESTS
4066 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4067#endif
4068 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4069#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4070 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4071 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4072 {
4073 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4074 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS;
4075 }
4076#else
4077 /* 64-bit host or hybrid host. */
4078 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4079#endif
4080 }
4081 else
4082 {
4083 /* Guest is not in long mode, use the 32-bit handler. */
4084#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4085 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4086 {
4087 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4088 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS;
4089 }
4090#else
4091 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4092#endif
4093 }
4094 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4095 return VINF_SUCCESS;
4096}
4097
4098
4099/**
4100 * Wrapper for running the guest code in VT-x.
4101 *
4102 * @returns VBox strict status code.
4103 * @param pVM Pointer to the VM.
4104 * @param pVCpu Pointer to the VMCPU.
4105 * @param pCtx Pointer to the guest-CPU context.
4106 *
4107 * @remarks No-long-jump zone!!!
4108 */
4109DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4110{
4111 /*
4112 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4113 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4114 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4115 */
4116 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4117 /** @todo Add stats for resume vs launch. */
4118#ifdef VBOX_WITH_KERNEL_USING_XMM
4119 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4120#else
4121 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4122#endif
4123}
4124
4125
4126/**
4127 * Reports world-switch error and dumps some useful debug info.
4128 *
4129 * @param pVM Pointer to the VM.
4130 * @param pVCpu Pointer to the VMCPU.
4131 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4132 * @param pCtx Pointer to the guest-CPU context.
4133 * @param pVmxTransient Pointer to the VMX transient structure (only
4134 * exitReason updated).
4135 */
4136static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4137{
4138 Assert(pVM);
4139 Assert(pVCpu);
4140 Assert(pCtx);
4141 Assert(pVmxTransient);
4142 HMVMX_ASSERT_PREEMPT_SAFE();
4143
4144 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4145 switch (rcVMRun)
4146 {
4147 case VERR_VMX_INVALID_VMXON_PTR:
4148 AssertFailed();
4149 break;
4150 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4151 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4152 {
4153 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4154 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4155 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4156 AssertRC(rc);
4157
4158 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4159 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4160 Cannot do it here as we may have been long preempted. */
4161
4162#ifdef VBOX_STRICT
4163 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4164 pVmxTransient->uExitReason));
4165 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4166 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4167 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4168 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4169 else
4170 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4171 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4172 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4173
4174 /* VMX control bits. */
4175 uint32_t u32Val;
4176 uint64_t u64Val;
4177 HMVMXHCUINTREG uHCReg;
4178 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4179 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4180 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4181 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4182 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4183 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4184 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4185 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4186 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4187 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4188 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4189 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4190 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4191 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4192 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4193 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4194 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4195 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4196 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4197 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4198 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4199 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4200 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4201 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4202 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4203 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4204 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4205 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4206 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4207 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4208 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4209 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4210 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4211 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4212 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4213 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4214 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4215 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4216 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4217 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4218 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4219 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4220
4221 /* Guest bits. */
4222 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4223 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4224 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4225 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4226 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4227 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4228 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4229 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4230
4231 /* Host bits. */
4232 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4233 Log4(("Host CR0 %#RHr\n", uHCReg));
4234 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4235 Log4(("Host CR3 %#RHr\n", uHCReg));
4236 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4237 Log4(("Host CR4 %#RHr\n", uHCReg));
4238
4239 RTGDTR HostGdtr;
4240 PCX86DESCHC pDesc;
4241 ASMGetGDTR(&HostGdtr);
4242 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4243 Log4(("Host CS %#08x\n", u32Val));
4244 if (u32Val < HostGdtr.cbGdt)
4245 {
4246 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4247 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4248 }
4249
4250 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4251 Log4(("Host DS %#08x\n", u32Val));
4252 if (u32Val < HostGdtr.cbGdt)
4253 {
4254 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4255 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4256 }
4257
4258 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4259 Log4(("Host ES %#08x\n", u32Val));
4260 if (u32Val < HostGdtr.cbGdt)
4261 {
4262 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4263 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4264 }
4265
4266 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4267 Log4(("Host FS %#08x\n", u32Val));
4268 if (u32Val < HostGdtr.cbGdt)
4269 {
4270 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4271 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4272 }
4273
4274 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4275 Log4(("Host GS %#08x\n", u32Val));
4276 if (u32Val < HostGdtr.cbGdt)
4277 {
4278 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4279 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4280 }
4281
4282 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4283 Log4(("Host SS %#08x\n", u32Val));
4284 if (u32Val < HostGdtr.cbGdt)
4285 {
4286 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4287 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4288 }
4289
4290 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4291 Log4(("Host TR %#08x\n", u32Val));
4292 if (u32Val < HostGdtr.cbGdt)
4293 {
4294 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4295 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4296 }
4297
4298 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4299 Log4(("Host TR Base %#RHv\n", uHCReg));
4300 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4301 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4302 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4303 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4304 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4305 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4306 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4307 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4308 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4309 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4310 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4311 Log4(("Host RSP %#RHv\n", uHCReg));
4312 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4313 Log4(("Host RIP %#RHv\n", uHCReg));
4314# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4315 if (HMVMX_IS_64BIT_HOST_MODE())
4316 {
4317 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4318 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4319 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4320 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4321 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4322 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4323 }
4324# endif
4325#endif /* VBOX_STRICT */
4326 break;
4327 }
4328
4329 default:
4330 /* Impossible */
4331 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4332 break;
4333 }
4334 NOREF(pVM);
4335}
4336
4337
4338#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4339#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4340# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4341#endif
4342#ifdef VBOX_STRICT
4343static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4344{
4345 switch (idxField)
4346 {
4347 case VMX_VMCS_GUEST_RIP:
4348 case VMX_VMCS_GUEST_RSP:
4349 case VMX_VMCS_GUEST_SYSENTER_EIP:
4350 case VMX_VMCS_GUEST_SYSENTER_ESP:
4351 case VMX_VMCS_GUEST_GDTR_BASE:
4352 case VMX_VMCS_GUEST_IDTR_BASE:
4353 case VMX_VMCS_GUEST_CS_BASE:
4354 case VMX_VMCS_GUEST_DS_BASE:
4355 case VMX_VMCS_GUEST_ES_BASE:
4356 case VMX_VMCS_GUEST_FS_BASE:
4357 case VMX_VMCS_GUEST_GS_BASE:
4358 case VMX_VMCS_GUEST_SS_BASE:
4359 case VMX_VMCS_GUEST_LDTR_BASE:
4360 case VMX_VMCS_GUEST_TR_BASE:
4361 case VMX_VMCS_GUEST_CR3:
4362 return true;
4363 }
4364 return false;
4365}
4366
4367static bool hmR0VmxIsValidReadField(uint32_t idxField)
4368{
4369 switch (idxField)
4370 {
4371 /* Read-only fields. */
4372 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4373 return true;
4374 }
4375 /* Remaining readable fields should also be writable. */
4376 return hmR0VmxIsValidWriteField(idxField);
4377}
4378#endif /* VBOX_STRICT */
4379
4380
4381/**
4382 * Executes the specified handler in 64-bit mode.
4383 *
4384 * @returns VBox status code.
4385 * @param pVM Pointer to the VM.
4386 * @param pVCpu Pointer to the VMCPU.
4387 * @param pCtx Pointer to the guest CPU context.
4388 * @param enmOp The operation to perform.
4389 * @param cbParam Number of parameters.
4390 * @param paParam Array of 32-bit parameters.
4391 */
4392VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4393 uint32_t *paParam)
4394{
4395 int rc, rc2;
4396 PHMGLOBALCPUINFO pCpu;
4397 RTHCPHYS HCPhysCpuPage;
4398 RTCCUINTREG uOldEflags;
4399
4400 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4401 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4402 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4403 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4404
4405#ifdef VBOX_STRICT
4406 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4407 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4408
4409 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4410 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4411#endif
4412
4413 /* Disable interrupts. */
4414 uOldEflags = ASMIntDisableFlags();
4415
4416#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4417 RTCPUID idHostCpu = RTMpCpuId();
4418 CPUMR0SetLApic(pVCpu, idHostCpu);
4419#endif
4420
4421 pCpu = HMR0GetCurrentCpu();
4422 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4423
4424 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4425 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4426
4427 /* Leave VMX Root Mode. */
4428 VMXDisable();
4429
4430 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4431
4432 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4433 CPUMSetHyperEIP(pVCpu, enmOp);
4434 for (int i = (int)cbParam - 1; i >= 0; i--)
4435 CPUMPushHyper(pVCpu, paParam[i]);
4436
4437 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4438
4439 /* Call the switcher. */
4440 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4441 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4442
4443 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4444 /* Make sure the VMX instructions don't cause #UD faults. */
4445 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4446
4447 /* Re-enter VMX Root Mode */
4448 rc2 = VMXEnable(HCPhysCpuPage);
4449 if (RT_FAILURE(rc2))
4450 {
4451 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4452 ASMSetFlags(uOldEflags);
4453 return rc2;
4454 }
4455
4456 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4457 AssertRC(rc2);
4458 Assert(!(ASMGetFlags() & X86_EFL_IF));
4459 ASMSetFlags(uOldEflags);
4460 return rc;
4461}
4462
4463
4464/**
4465 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4466 * supporting 64-bit guests.
4467 *
4468 * @returns VBox status code.
4469 * @param fResume Whether to VMLAUNCH or VMRESUME.
4470 * @param pCtx Pointer to the guest-CPU context.
4471 * @param pCache Pointer to the VMCS cache.
4472 * @param pVM Pointer to the VM.
4473 * @param pVCpu Pointer to the VMCPU.
4474 */
4475DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4476{
4477 uint32_t aParam[6];
4478 PHMGLOBALCPUINFO pCpu = NULL;
4479 RTHCPHYS HCPhysCpuPage = 0;
4480 int rc = VERR_INTERNAL_ERROR_5;
4481
4482 pCpu = HMR0GetCurrentCpu();
4483 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4484
4485#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4486 pCache->uPos = 1;
4487 pCache->interPD = PGMGetInterPaeCR3(pVM);
4488 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4489#endif
4490
4491#ifdef VBOX_STRICT
4492 pCache->TestIn.HCPhysCpuPage = 0;
4493 pCache->TestIn.HCPhysVmcs = 0;
4494 pCache->TestIn.pCache = 0;
4495 pCache->TestOut.HCPhysVmcs = 0;
4496 pCache->TestOut.pCache = 0;
4497 pCache->TestOut.pCtx = 0;
4498 pCache->TestOut.eflags = 0;
4499#endif
4500
4501 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4502 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4503 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4504 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4505 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4506 aParam[5] = 0;
4507
4508#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4509 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4510 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4511#endif
4512 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4513
4514#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4515 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4516 Assert(pCtx->dr[4] == 10);
4517 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4518#endif
4519
4520#ifdef VBOX_STRICT
4521 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4522 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4523 pVCpu->hm.s.vmx.HCPhysVmcs));
4524 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4525 pCache->TestOut.HCPhysVmcs));
4526 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4527 pCache->TestOut.pCache));
4528 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4529 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4530 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4531 pCache->TestOut.pCtx));
4532 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4533#endif
4534 return rc;
4535}
4536
4537
4538/**
4539 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4540 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4541 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4542 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4543 *
4544 * @returns VBox status code.
4545 * @param pVM Pointer to the VM.
4546 * @param pVCpu Pointer to the VMCPU.
4547 */
4548static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4549{
4550#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4551{ \
4552 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4553 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4554 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4555 ++cReadFields; \
4556}
4557
4558 AssertPtr(pVM);
4559 AssertPtr(pVCpu);
4560 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4561 uint32_t cReadFields = 0;
4562
4563 /*
4564 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4565 * and serve to indicate exceptions to the rules.
4566 */
4567
4568 /* Guest-natural selector base fields. */
4569#if 0
4570 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4571 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4572 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4573#endif
4574 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4575 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4576 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4577 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4578 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4579 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4580 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4581 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4582 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4583 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4584 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4585 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4586#if 0
4587 /* Unused natural width guest-state fields. */
4588 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4589 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4590#endif
4591 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4592 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4593
4594 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4595#if 0
4596 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4597 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4598 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4599 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4600 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4601 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4602 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4603 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4604 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4605#endif
4606
4607 /* Natural width guest-state fields. */
4608 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4609#if 0
4610 /* Currently unused field. */
4611 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4612#endif
4613
4614 if (pVM->hm.s.fNestedPaging)
4615 {
4616 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4617 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4618 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4619 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4620 }
4621 else
4622 {
4623 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4624 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4625 }
4626
4627#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4628 return VINF_SUCCESS;
4629}
4630
4631
4632/**
4633 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4634 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4635 * darwin, running 64-bit guests).
4636 *
4637 * @returns VBox status code.
4638 * @param pVCpu Pointer to the VMCPU.
4639 * @param idxField The VMCS field encoding.
4640 * @param u64Val 16, 32 or 64 bits value.
4641 */
4642VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4643{
4644 int rc;
4645 switch (idxField)
4646 {
4647 /*
4648 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4649 */
4650 /* 64-bit Control fields. */
4651 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4652 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4653 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4654 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4655 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4656 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4657 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4658 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4659 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4660 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4661 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4662 case VMX_VMCS64_CTRL_EPTP_FULL:
4663 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4664 /* 64-bit Guest-state fields. */
4665 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4666 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4667 case VMX_VMCS64_GUEST_PAT_FULL:
4668 case VMX_VMCS64_GUEST_EFER_FULL:
4669 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4670 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4671 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4672 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4673 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4674 /* 64-bit Host-state fields. */
4675 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4676 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4677 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4678 {
4679 rc = VMXWriteVmcs32(idxField, u64Val);
4680 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4681 break;
4682 }
4683
4684 /*
4685 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4686 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4687 */
4688 /* Natural-width Guest-state fields. */
4689 case VMX_VMCS_GUEST_CR3:
4690 case VMX_VMCS_GUEST_ES_BASE:
4691 case VMX_VMCS_GUEST_CS_BASE:
4692 case VMX_VMCS_GUEST_SS_BASE:
4693 case VMX_VMCS_GUEST_DS_BASE:
4694 case VMX_VMCS_GUEST_FS_BASE:
4695 case VMX_VMCS_GUEST_GS_BASE:
4696 case VMX_VMCS_GUEST_LDTR_BASE:
4697 case VMX_VMCS_GUEST_TR_BASE:
4698 case VMX_VMCS_GUEST_GDTR_BASE:
4699 case VMX_VMCS_GUEST_IDTR_BASE:
4700 case VMX_VMCS_GUEST_RSP:
4701 case VMX_VMCS_GUEST_RIP:
4702 case VMX_VMCS_GUEST_SYSENTER_ESP:
4703 case VMX_VMCS_GUEST_SYSENTER_EIP:
4704 {
4705 if (!(u64Val >> 32))
4706 {
4707 /* If this field is 64-bit, VT-x will zero out the top bits. */
4708 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4709 }
4710 else
4711 {
4712 /* Assert that only the 32->64 switcher case should ever come here. */
4713 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4714 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4715 }
4716 break;
4717 }
4718
4719 default:
4720 {
4721 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4722 rc = VERR_INVALID_PARAMETER;
4723 break;
4724 }
4725 }
4726 AssertRCReturn(rc, rc);
4727 return rc;
4728}
4729
4730
4731/**
4732 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4733 * hosts (except darwin) for 64-bit guests.
4734 *
4735 * @param pVCpu Pointer to the VMCPU.
4736 * @param idxField The VMCS field encoding.
4737 * @param u64Val 16, 32 or 64 bits value.
4738 */
4739VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4740{
4741 AssertPtr(pVCpu);
4742 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4743
4744 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4745 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4746
4747 /* Make sure there are no duplicates. */
4748 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4749 {
4750 if (pCache->Write.aField[i] == idxField)
4751 {
4752 pCache->Write.aFieldVal[i] = u64Val;
4753 return VINF_SUCCESS;
4754 }
4755 }
4756
4757 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4758 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4759 pCache->Write.cValidEntries++;
4760 return VINF_SUCCESS;
4761}
4762
4763/* Enable later when the assembly code uses these as callbacks. */
4764#if 0
4765/*
4766 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4767 *
4768 * @param pVCpu Pointer to the VMCPU.
4769 * @param pCache Pointer to the VMCS cache.
4770 *
4771 * @remarks No-long-jump zone!!!
4772 */
4773VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4774{
4775 AssertPtr(pCache);
4776 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4777 {
4778 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4779 AssertRC(rc);
4780 }
4781 pCache->Write.cValidEntries = 0;
4782}
4783
4784
4785/**
4786 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
4787 *
4788 * @param pVCpu Pointer to the VMCPU.
4789 * @param pCache Pointer to the VMCS cache.
4790 *
4791 * @remarks No-long-jump zone!!!
4792 */
4793VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4794{
4795 AssertPtr(pCache);
4796 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4797 {
4798 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4799 AssertRC(rc);
4800 }
4801}
4802#endif
4803#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4804
4805
4806/**
4807 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4808 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4809 * timer.
4810 *
4811 * @returns VBox status code.
4812 * @param pVCpu Pointer to the VMCPU.
4813 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4814 * out-of-sync. Make sure to update the required fields
4815 * before using them.
4816 * @remarks No-long-jump zone!!!
4817 */
4818static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4819{
4820 int rc = VERR_INTERNAL_ERROR_5;
4821 bool fOffsettedTsc = false;
4822 PVM pVM = pVCpu->CTX_SUFF(pVM);
4823 if (pVM->hm.s.vmx.fUsePreemptTimer)
4824 {
4825 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4826
4827 /* Make sure the returned values have sane upper and lower boundaries. */
4828 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4829 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4830 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4831 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4832
4833 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4834 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4835 }
4836 else
4837 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4838
4839 if (fOffsettedTsc)
4840 {
4841 uint64_t u64CurTSC = ASMReadTSC();
4842 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4843 {
4844 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4845 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4846
4847 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4848 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4849 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4850 }
4851 else
4852 {
4853 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4854 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4855 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4856 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4857 }
4858 }
4859 else
4860 {
4861 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4862 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4863 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4864 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4865 }
4866}
4867
4868
4869/**
4870 * Determines if an exception is a contributory exception. Contributory
4871 * exceptions are ones which can cause double-faults. Page-fault is
4872 * intentionally not included here as it's a conditional contributory exception.
4873 *
4874 * @returns true if the exception is contributory, false otherwise.
4875 * @param uVector The exception vector.
4876 */
4877DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4878{
4879 switch (uVector)
4880 {
4881 case X86_XCPT_GP:
4882 case X86_XCPT_SS:
4883 case X86_XCPT_NP:
4884 case X86_XCPT_TS:
4885 case X86_XCPT_DE:
4886 return true;
4887 default:
4888 break;
4889 }
4890 return false;
4891}
4892
4893
4894/**
4895 * Sets an event as a pending event to be injected into the guest.
4896 *
4897 * @param pVCpu Pointer to the VMCPU.
4898 * @param u32IntrInfo The VM-entry interruption-information field.
4899 * @param cbInstr The VM-entry instruction length in bytes (for software
4900 * interrupts, exceptions and privileged software
4901 * exceptions).
4902 * @param u32ErrCode The VM-entry exception error code.
4903 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4904 * page-fault.
4905 *
4906 * @remarks Statistics counter assumes this is a guest event being injected or
4907 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4908 * always incremented.
4909 */
4910DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4911 RTGCUINTPTR GCPtrFaultAddress)
4912{
4913 Assert(!pVCpu->hm.s.Event.fPending);
4914 pVCpu->hm.s.Event.fPending = true;
4915 pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4916 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4917 pVCpu->hm.s.Event.cbInstr = cbInstr;
4918 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4919
4920 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4921}
4922
4923
4924/**
4925 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4926 *
4927 * @param pVCpu Pointer to the VMCPU.
4928 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4929 * out-of-sync. Make sure to update the required fields
4930 * before using them.
4931 */
4932DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4933{
4934 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
4935 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4936 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4937 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
4938}
4939
4940
4941/**
4942 * Handle a condition that occurred while delivering an event through the guest
4943 * IDT.
4944 *
4945 * @returns VBox status code (informational error codes included).
4946 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
4947 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4948 * continue execution of the guest which will delivery the #DF.
4949 * @retval VINF_EM_RESET if we detected a triple-fault condition.
4950 *
4951 * @param pVCpu Pointer to the VMCPU.
4952 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4953 * out-of-sync. Make sure to update the required fields
4954 * before using them.
4955 * @param pVmxTransient Pointer to the VMX transient structure.
4956 *
4957 * @remarks No-long-jump zone!!!
4958 */
4959static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4960{
4961 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4962 AssertRCReturn(rc, rc);
4963 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4964 {
4965 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4966 AssertRCReturn(rc, rc);
4967
4968 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4969 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4970 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4971
4972 typedef enum
4973 {
4974 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4975 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4976 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4977 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4978 } VMXREFLECTXCPT;
4979
4980 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4981 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4982 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4983 {
4984 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4985 {
4986 enmReflect = VMXREFLECTXCPT_XCPT;
4987#ifdef VBOX_STRICT
4988 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4989 && uExitVector == X86_XCPT_PF)
4990 {
4991 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4992 }
4993#endif
4994 if ( uExitVector == X86_XCPT_PF
4995 && uIdtVector == X86_XCPT_PF)
4996 {
4997 pVmxTransient->fVectoringPF = true;
4998 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4999 }
5000 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5001 && hmR0VmxIsContributoryXcpt(uExitVector)
5002 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5003 || uIdtVector == X86_XCPT_PF))
5004 {
5005 enmReflect = VMXREFLECTXCPT_DF;
5006 }
5007 else if (uIdtVector == X86_XCPT_DF)
5008 enmReflect = VMXREFLECTXCPT_TF;
5009 }
5010 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5011 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5012 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5013 {
5014 /*
5015 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5016 * (whatever they are) as they reoccur when restarting the instruction.
5017 */
5018 enmReflect = VMXREFLECTXCPT_XCPT;
5019 }
5020 }
5021 else
5022 {
5023 /*
5024 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5025 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5026 * original exception to the guest after handling the VM-exit.
5027 */
5028 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5029 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5030 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5031 {
5032 enmReflect = VMXREFLECTXCPT_XCPT;
5033 }
5034 }
5035
5036 switch (enmReflect)
5037 {
5038 case VMXREFLECTXCPT_XCPT:
5039 {
5040 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5041 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5042 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5043
5044 uint32_t u32ErrCode = 0;
5045 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5046 {
5047 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5048 AssertRCReturn(rc, rc);
5049 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5050 }
5051
5052 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5053 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5054 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5055 rc = VINF_SUCCESS;
5056 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5057 pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
5058
5059 break;
5060 }
5061
5062 case VMXREFLECTXCPT_DF:
5063 {
5064 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5065 rc = VINF_HM_DOUBLE_FAULT;
5066 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5067 pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
5068
5069 break;
5070 }
5071
5072 case VMXREFLECTXCPT_TF:
5073 {
5074 rc = VINF_EM_RESET;
5075 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5076 uExitVector));
5077 break;
5078 }
5079
5080 default:
5081 Assert(rc == VINF_SUCCESS);
5082 break;
5083 }
5084 }
5085 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5086 return rc;
5087}
5088
5089
5090/**
5091 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5092 *
5093 * @returns VBox status code.
5094 * @param pVCpu Pointer to the VMCPU.
5095 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5096 * out-of-sync. Make sure to update the required fields
5097 * before using them.
5098 *
5099 * @remarks No-long-jump zone!!!
5100 */
5101static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5102{
5103 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5104 {
5105 uint32_t uVal = 0;
5106 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5107 AssertRCReturn(rc, rc);
5108 uint32_t uShadow = 0;
5109 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5110 AssertRCReturn(rc, rc);
5111
5112 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5113 CPUMSetGuestCR0(pVCpu, uVal);
5114 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5115 }
5116 return VINF_SUCCESS;
5117}
5118
5119
5120/**
5121 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5122 *
5123 * @returns VBox status code.
5124 * @param pVCpu Pointer to the VMCPU.
5125 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5126 * out-of-sync. Make sure to update the required fields
5127 * before using them.
5128 *
5129 * @remarks No-long-jump zone!!!
5130 */
5131static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5132{
5133 int rc = VINF_SUCCESS;
5134 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5135 {
5136 uint32_t uVal = 0;
5137 uint32_t uShadow = 0;
5138 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5139 AssertRCReturn(rc, rc);
5140 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5141 AssertRCReturn(rc, rc);
5142
5143 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5144 CPUMSetGuestCR4(pVCpu, uVal);
5145 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5146 }
5147 return rc;
5148}
5149
5150
5151/**
5152 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5153 *
5154 * @returns VBox status code.
5155 * @param pVCpu Pointer to the VMCPU.
5156 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5157 * out-of-sync. Make sure to update the required fields
5158 * before using them.
5159 *
5160 * @remarks No-long-jump zone!!!
5161 */
5162static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5163{
5164 int rc = VINF_SUCCESS;
5165 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5166 {
5167 uint64_t u64Val = 0;
5168 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5169 AssertRCReturn(rc, rc);
5170
5171 pMixedCtx->rip = u64Val;
5172 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5173 }
5174 return rc;
5175}
5176
5177
5178/**
5179 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5180 *
5181 * @returns VBox status code.
5182 * @param pVCpu Pointer to the VMCPU.
5183 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5184 * out-of-sync. Make sure to update the required fields
5185 * before using them.
5186 *
5187 * @remarks No-long-jump zone!!!
5188 */
5189static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5190{
5191 int rc = VINF_SUCCESS;
5192 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5193 {
5194 uint64_t u64Val = 0;
5195 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5196 AssertRCReturn(rc, rc);
5197
5198 pMixedCtx->rsp = u64Val;
5199 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5200 }
5201 return rc;
5202}
5203
5204
5205/**
5206 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5207 *
5208 * @returns VBox status code.
5209 * @param pVCpu Pointer to the VMCPU.
5210 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5211 * out-of-sync. Make sure to update the required fields
5212 * before using them.
5213 *
5214 * @remarks No-long-jump zone!!!
5215 */
5216static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5217{
5218 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5219 {
5220 uint32_t uVal = 0;
5221 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5222 AssertRCReturn(rc, rc);
5223
5224 pMixedCtx->eflags.u32 = uVal;
5225 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5226 {
5227 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5228 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5229
5230 pMixedCtx->eflags.Bits.u1VM = 0;
5231 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5232 }
5233
5234 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5235 }
5236 return VINF_SUCCESS;
5237}
5238
5239
5240/**
5241 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5242 * guest-CPU context.
5243 */
5244DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5245{
5246 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5247 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5248 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5249 return rc;
5250}
5251
5252
5253/**
5254 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5255 * from the guest-state area in the VMCS.
5256 *
5257 * @param pVCpu Pointer to the VMCPU.
5258 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5259 * out-of-sync. Make sure to update the required fields
5260 * before using them.
5261 *
5262 * @remarks No-long-jump zone!!!
5263 */
5264static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5265{
5266 uint32_t uIntrState = 0;
5267 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5268 AssertRC(rc);
5269
5270 if (!uIntrState)
5271 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5272 else
5273 {
5274 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5275 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5276 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5277 AssertRC(rc);
5278 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5279 AssertRC(rc);
5280
5281 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5282 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5283 }
5284}
5285
5286
5287/**
5288 * Saves the guest's activity state.
5289 *
5290 * @returns VBox status code.
5291 * @param pVCpu Pointer to the VMCPU.
5292 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5293 * out-of-sync. Make sure to update the required fields
5294 * before using them.
5295 *
5296 * @remarks No-long-jump zone!!!
5297 */
5298static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5299{
5300 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5301 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5302 return VINF_SUCCESS;
5303}
5304
5305
5306/**
5307 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5308 * the current VMCS into the guest-CPU context.
5309 *
5310 * @returns VBox status code.
5311 * @param pVCpu Pointer to the VMCPU.
5312 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5313 * out-of-sync. Make sure to update the required fields
5314 * before using them.
5315 *
5316 * @remarks No-long-jump zone!!!
5317 */
5318static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5319{
5320 int rc = VINF_SUCCESS;
5321 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5322 {
5323 uint32_t u32Val = 0;
5324 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5325 pMixedCtx->SysEnter.cs = u32Val;
5326 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5327 }
5328
5329 uint64_t u64Val = 0;
5330 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5331 {
5332 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5333 pMixedCtx->SysEnter.eip = u64Val;
5334 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5335 }
5336 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5337 {
5338 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5339 pMixedCtx->SysEnter.esp = u64Val;
5340 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5341 }
5342 return rc;
5343}
5344
5345
5346/**
5347 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5348 * context.
5349 *
5350 * @returns VBox status code.
5351 * @param pVCpu Pointer to the VMCPU.
5352 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5353 * out-of-sync. Make sure to update the required fields
5354 * before using them.
5355 *
5356 * @remarks No-long-jump zone!!!
5357 */
5358static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5359{
5360 int rc = VINF_SUCCESS;
5361 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5362 {
5363 uint64_t u64Val = 0;
5364 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5365 pMixedCtx->fs.u64Base = u64Val;
5366 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5367 }
5368 return rc;
5369}
5370
5371
5372/**
5373 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5374 * context.
5375 *
5376 * @returns VBox status code.
5377 * @param pVCpu Pointer to the VMCPU.
5378 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5379 * out-of-sync. Make sure to update the required fields
5380 * before using them.
5381 *
5382 * @remarks No-long-jump zone!!!
5383 */
5384static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5385{
5386 int rc = VINF_SUCCESS;
5387 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5388 {
5389 uint64_t u64Val = 0;
5390 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5391 pMixedCtx->gs.u64Base = u64Val;
5392 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5393 }
5394 return rc;
5395}
5396
5397
5398/**
5399 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5400 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5401 * and TSC_AUX.
5402 *
5403 * @returns VBox status code.
5404 * @param pVCpu Pointer to the VMCPU.
5405 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5406 * out-of-sync. Make sure to update the required fields
5407 * before using them.
5408 *
5409 * @remarks No-long-jump zone!!!
5410 */
5411static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5412{
5413 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5414 return VINF_SUCCESS;
5415
5416#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5417 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5418 {
5419 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5420 pMsr += i;
5421 switch (pMsr->u32Msr)
5422 {
5423 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5424 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5425 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5426 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5427 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5428 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5429 default:
5430 {
5431 AssertFailed();
5432 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5433 }
5434 }
5435 }
5436#endif
5437
5438 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5439 return VINF_SUCCESS;
5440}
5441
5442
5443/**
5444 * Saves the guest control registers from the current VMCS into the guest-CPU
5445 * context.
5446 *
5447 * @returns VBox status code.
5448 * @param pVCpu Pointer to the VMCPU.
5449 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5450 * out-of-sync. Make sure to update the required fields
5451 * before using them.
5452 *
5453 * @remarks No-long-jump zone!!!
5454 */
5455static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5456{
5457 /* Guest CR0. Guest FPU. */
5458 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5459 AssertRCReturn(rc, rc);
5460
5461 /* Guest CR4. */
5462 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5463 AssertRCReturn(rc, rc);
5464
5465 /* Guest CR2 - updated always during the world-switch or in #PF. */
5466 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5467 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5468 {
5469 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5470 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5471
5472 PVM pVM = pVCpu->CTX_SUFF(pVM);
5473 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5474 || ( pVM->hm.s.fNestedPaging
5475 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5476 {
5477 uint64_t u64Val = 0;
5478 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5479 if (pMixedCtx->cr3 != u64Val)
5480 {
5481 CPUMSetGuestCR3(pVCpu, u64Val);
5482 if (VMMRZCallRing3IsEnabled(pVCpu))
5483 {
5484 PGMUpdateCR3(pVCpu, u64Val);
5485 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5486 }
5487 else
5488 {
5489 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5490 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5491 }
5492 }
5493
5494 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5495 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5496 {
5497 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5498 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5499 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5500 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5501
5502 if (VMMRZCallRing3IsEnabled(pVCpu))
5503 {
5504 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5505 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5506 }
5507 else
5508 {
5509 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5510 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5511 }
5512 }
5513 }
5514
5515 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5516 }
5517
5518 /*
5519 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5520 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5521 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5522 *
5523 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5524 */
5525 if (VMMRZCallRing3IsEnabled(pVCpu))
5526 {
5527 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5528 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5529
5530 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5531 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5532
5533 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5534 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5535 }
5536
5537 return rc;
5538}
5539
5540
5541/**
5542 * Reads a guest segment register from the current VMCS into the guest-CPU
5543 * context.
5544 *
5545 * @returns VBox status code.
5546 * @param pVCpu Pointer to the VMCPU.
5547 * @param idxSel Index of the selector in the VMCS.
5548 * @param idxLimit Index of the segment limit in the VMCS.
5549 * @param idxBase Index of the segment base in the VMCS.
5550 * @param idxAccess Index of the access rights of the segment in the VMCS.
5551 * @param pSelReg Pointer to the segment selector.
5552 *
5553 * @remarks No-long-jump zone!!!
5554 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5555 * macro as that takes care of whether to read from the VMCS cache or
5556 * not.
5557 */
5558DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5559 PCPUMSELREG pSelReg)
5560{
5561 uint32_t u32Val = 0;
5562 int rc = VMXReadVmcs32(idxSel, &u32Val);
5563 AssertRCReturn(rc, rc);
5564 pSelReg->Sel = (uint16_t)u32Val;
5565 pSelReg->ValidSel = (uint16_t)u32Val;
5566 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5567
5568 rc = VMXReadVmcs32(idxLimit, &u32Val);
5569 AssertRCReturn(rc, rc);
5570 pSelReg->u32Limit = u32Val;
5571
5572 uint64_t u64Val = 0;
5573 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5574 AssertRCReturn(rc, rc);
5575 pSelReg->u64Base = u64Val;
5576
5577 rc = VMXReadVmcs32(idxAccess, &u32Val);
5578 AssertRCReturn(rc, rc);
5579 pSelReg->Attr.u = u32Val;
5580
5581 /*
5582 * If VT-x marks the segment as unusable, most other bits remain undefined:
5583 * - For CS the L, D and G bits have meaning.
5584 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5585 * - For the remaining data segments no bits are defined.
5586 *
5587 * The present bit and the unusable bit has been observed to be set at the
5588 * same time (the selector was supposed to invalid as we started executing
5589 * a V8086 interrupt in ring-0).
5590 *
5591 * What should be important for the rest of the VBox code that the P bit is
5592 * cleared. Some of the other VBox code recognizes the unusable bit, but
5593 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5594 * safe side here, we'll strip off P and other bits we don't care about. If
5595 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5596 *
5597 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5598 */
5599 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5600 {
5601 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5602
5603 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5604 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5605 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5606
5607 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5608#ifdef DEBUG_bird
5609 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5610 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5611 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5612#endif
5613 }
5614 return VINF_SUCCESS;
5615}
5616
5617
5618#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5619# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5620 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5621 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5622#else
5623# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5624 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5625 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5626#endif
5627
5628
5629/**
5630 * Saves the guest segment registers from the current VMCS into the guest-CPU
5631 * context.
5632 *
5633 * @returns VBox status code.
5634 * @param pVCpu Pointer to the VMCPU.
5635 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5636 * out-of-sync. Make sure to update the required fields
5637 * before using them.
5638 *
5639 * @remarks No-long-jump zone!!!
5640 */
5641static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5642{
5643 /* Guest segment registers. */
5644 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5645 {
5646 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5647 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5648 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5649 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5650 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5651 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5652 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5653
5654 /* Restore segment attributes for real-on-v86 mode hack. */
5655 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5656 {
5657 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5658 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5659 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5660 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5661 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5662 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5663 }
5664 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5665 }
5666
5667 return VINF_SUCCESS;
5668}
5669
5670
5671/**
5672 * Saves the guest descriptor table registers and task register from the current
5673 * VMCS into the guest-CPU context.
5674 *
5675 * @returns VBox status code.
5676 * @param pVCpu Pointer to the VMCPU.
5677 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5678 * out-of-sync. Make sure to update the required fields
5679 * before using them.
5680 *
5681 * @remarks No-long-jump zone!!!
5682 */
5683static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5684{
5685 int rc = VINF_SUCCESS;
5686
5687 /* Guest LDTR. */
5688 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5689 {
5690 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5691 AssertRCReturn(rc, rc);
5692 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5693 }
5694
5695 /* Guest GDTR. */
5696 uint64_t u64Val = 0;
5697 uint32_t u32Val = 0;
5698 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5699 {
5700 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5701 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5702 pMixedCtx->gdtr.pGdt = u64Val;
5703 pMixedCtx->gdtr.cbGdt = u32Val;
5704 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
5705 }
5706
5707 /* Guest IDTR. */
5708 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5709 {
5710 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5711 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5712 pMixedCtx->idtr.pIdt = u64Val;
5713 pMixedCtx->idtr.cbIdt = u32Val;
5714 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
5715 }
5716
5717 /* Guest TR. */
5718 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5719 {
5720 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5721 AssertRCReturn(rc, rc);
5722
5723 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5724 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5725 {
5726 rc = VMXLOCAL_READ_SEG(TR, tr);
5727 AssertRCReturn(rc, rc);
5728 }
5729 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
5730 }
5731 return rc;
5732}
5733
5734#undef VMXLOCAL_READ_SEG
5735
5736
5737/**
5738 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5739 * context.
5740 *
5741 * @returns VBox status code.
5742 * @param pVCpu Pointer to the VMCPU.
5743 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5744 * out-of-sync. Make sure to update the required fields
5745 * before using them.
5746 *
5747 * @remarks No-long-jump zone!!!
5748 */
5749static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5750{
5751 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5752 {
5753 if (!CPUMIsHyperDebugStateActive(pVCpu))
5754 {
5755 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5756 uint32_t u32Val;
5757 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5758 pMixedCtx->dr[7] = u32Val;
5759 }
5760
5761 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
5762 }
5763 return VINF_SUCCESS;
5764}
5765
5766
5767/**
5768 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
5769 *
5770 * @returns VBox status code.
5771 * @param pVCpu Pointer to the VMCPU.
5772 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5773 * out-of-sync. Make sure to update the required fields
5774 * before using them.
5775 *
5776 * @remarks No-long-jump zone!!!
5777 */
5778static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5779{
5780 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5781 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
5782 return VINF_SUCCESS;
5783}
5784
5785
5786/**
5787 * Saves the entire guest state from the currently active VMCS into the
5788 * guest-CPU context. This essentially VMREADs all guest-data.
5789 *
5790 * @returns VBox status code.
5791 * @param pVCpu Pointer to the VMCPU.
5792 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5793 * out-of-sync. Make sure to update the required fields
5794 * before using them.
5795 */
5796static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5797{
5798 Assert(pVCpu);
5799 Assert(pMixedCtx);
5800
5801 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5802 return VINF_SUCCESS;
5803
5804 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5805 again on the ring-3 callback path, there is no real need to. */
5806 if (VMMRZCallRing3IsEnabled(pVCpu))
5807 VMMR0LogFlushDisable(pVCpu);
5808 else
5809 Assert(VMMR0IsLogFlushDisabled(pVCpu));
5810 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5811
5812 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5813 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5814
5815 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5816 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5817
5818 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5819 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5820
5821 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5822 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5823
5824 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5825 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5826
5827 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5828 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5829
5830 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5831 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5832
5833 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5834 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5835
5836 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5837 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5838
5839 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5840 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5841
5842 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5843 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5844
5845 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5846 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5847
5848 if (VMMRZCallRing3IsEnabled(pVCpu))
5849 VMMR0LogFlushEnable(pVCpu);
5850
5851 return rc;
5852}
5853
5854
5855/**
5856 * Check per-VM and per-VCPU force flag actions that require us to go back to
5857 * ring-3 for one reason or another.
5858 *
5859 * @returns VBox status code (information status code included).
5860 * @retval VINF_SUCCESS if we don't have any actions that require going back to
5861 * ring-3.
5862 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5863 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5864 * interrupts)
5865 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5866 * all EMTs to be in ring-3.
5867 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5868 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5869 * to the EM loop.
5870 *
5871 * @param pVM Pointer to the VM.
5872 * @param pVCpu Pointer to the VMCPU.
5873 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5874 * out-of-sync. Make sure to update the required fields
5875 * before using them.
5876 */
5877static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5878{
5879 Assert(VMMRZCallRing3IsEnabled(pVCpu));
5880
5881 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5882 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5883 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5884 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5885 {
5886 /* We need the control registers now, make sure the guest-CPU context is updated. */
5887 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5888 AssertRCReturn(rc3, rc3);
5889
5890 /* Pending HM CR3 sync. */
5891 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5892 {
5893 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5894 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
5895 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5896 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5897 }
5898
5899 /* Pending HM PAE PDPEs. */
5900 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5901 {
5902 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5903 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5904 }
5905
5906 /* Pending PGM C3 sync. */
5907 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5908 {
5909 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
5910 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5911 if (rc2 != VINF_SUCCESS)
5912 {
5913 AssertRC(rc2);
5914 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5915 return rc2;
5916 }
5917 }
5918
5919 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5920 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5921 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5922 {
5923 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5924 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5925 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5926 return rc2;
5927 }
5928
5929 /* Pending VM request packets, such as hardware interrupts. */
5930 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5931 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5932 {
5933 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5934 return VINF_EM_PENDING_REQUEST;
5935 }
5936
5937 /* Pending PGM pool flushes. */
5938 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5939 {
5940 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5941 return VINF_PGM_POOL_FLUSH_PENDING;
5942 }
5943
5944 /* Pending DMA requests. */
5945 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5946 {
5947 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5948 return VINF_EM_RAW_TO_R3;
5949 }
5950 }
5951
5952 return VINF_SUCCESS;
5953}
5954
5955
5956/**
5957 * Converts any TRPM trap into a pending HM event. This is typically used when
5958 * entering from ring-3 (not longjmp returns).
5959 *
5960 * @param pVCpu Pointer to the VMCPU.
5961 */
5962static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5963{
5964 Assert(TRPMHasTrap(pVCpu));
5965 Assert(!pVCpu->hm.s.Event.fPending);
5966
5967 uint8_t uVector;
5968 TRPMEVENT enmTrpmEvent;
5969 RTGCUINT uErrCode;
5970 RTGCUINTPTR GCPtrFaultAddress;
5971 uint8_t cbInstr;
5972
5973 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5974 AssertRC(rc);
5975
5976 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5977 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
5978 if (enmTrpmEvent == TRPM_TRAP)
5979 {
5980 switch (uVector)
5981 {
5982 case X86_XCPT_BP:
5983 case X86_XCPT_OF:
5984 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5985 break;
5986
5987 case X86_XCPT_PF:
5988 case X86_XCPT_DF:
5989 case X86_XCPT_TS:
5990 case X86_XCPT_NP:
5991 case X86_XCPT_SS:
5992 case X86_XCPT_GP:
5993 case X86_XCPT_AC:
5994 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5995 /* no break! */
5996 default:
5997 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5998 break;
5999 }
6000 }
6001 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6002 {
6003 if (uVector == X86_XCPT_NMI)
6004 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6005 else
6006 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6007 }
6008 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6009 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6010 else
6011 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6012
6013 rc = TRPMResetTrap(pVCpu);
6014 AssertRC(rc);
6015 Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6016 u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6017
6018 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6019 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6020}
6021
6022
6023/**
6024 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6025 * VT-x to execute any instruction.
6026 *
6027 * @param pvCpu Pointer to the VMCPU.
6028 */
6029static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6030{
6031 Assert(pVCpu->hm.s.Event.fPending);
6032
6033 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6034 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
6035 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
6036 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6037
6038 /* If a trap was already pending, we did something wrong! */
6039 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6040
6041 TRPMEVENT enmTrapType;
6042 switch (uVectorType)
6043 {
6044 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6045 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6046 enmTrapType = TRPM_HARDWARE_INT;
6047 break;
6048
6049 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6050 enmTrapType = TRPM_SOFTWARE_INT;
6051 break;
6052
6053 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6054 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6055 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6056 enmTrapType = TRPM_TRAP;
6057 break;
6058
6059 default:
6060 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6061 enmTrapType = TRPM_32BIT_HACK;
6062 break;
6063 }
6064
6065 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6066
6067 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6068 AssertRC(rc);
6069
6070 if (fErrorCodeValid)
6071 TRPMSetErrorCode(pVCpu, uErrorCode);
6072
6073 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6074 && uVector == X86_XCPT_PF)
6075 {
6076 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6077 }
6078 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6079 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6080 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6081 {
6082 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6083 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6084 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6085 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6086 }
6087 pVCpu->hm.s.Event.fPending = false;
6088}
6089
6090
6091/**
6092 * Does the necessary state syncing before returning to ring-3 for any reason
6093 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6094 *
6095 * @returns VBox status code.
6096 * @param pVM Pointer to the VM.
6097 * @param pVCpu Pointer to the VMCPU.
6098 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6099 * be out-of-sync. Make sure to update the required
6100 * fields before using them.
6101 * @param fSaveGuestState Whether to save the guest state or not.
6102 *
6103 * @remarks No-long-jmp zone!!!
6104 */
6105static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6106{
6107 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6108 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6109
6110 RTCPUID idCpu = RTMpCpuId();
6111 Log4Func(("HostCpuId=%u\n", idCpu));
6112
6113 /* Save the guest state if necessary. */
6114 if ( fSaveGuestState
6115 && pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6116 {
6117 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6118 AssertRCReturn(rc, rc);
6119 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6120 }
6121
6122 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6123 if (CPUMIsGuestFPUStateActive(pVCpu))
6124 {
6125 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6126 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6127 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
6128 }
6129
6130 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6131#ifdef VBOX_STRICT
6132 if (CPUMIsHyperDebugStateActive(pVCpu))
6133 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6134#endif
6135 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6136 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
6137 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6138 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6139
6140#if HC_ARCH_BITS == 64
6141 /* Restore host-state bits that VT-x only restores partially. */
6142 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6143 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6144 {
6145 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6146 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6147 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6148 }
6149#endif
6150
6151 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6152 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6153 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6154 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6155 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6156 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6157 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6158 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6159
6160 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6161
6162 /** @todo This kinda defeats the purpose of having preemption hooks.
6163 * The problem is, deregistering the hooks should be moved to a place that
6164 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6165 * context.
6166 */
6167 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6168 {
6169 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6170 AssertRCReturn(rc, rc);
6171
6172 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6173 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6174 }
6175 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6176 NOREF(idCpu);
6177
6178 return VINF_SUCCESS;
6179}
6180
6181
6182/**
6183 * Leaves the VT-x session.
6184 *
6185 * @returns VBox status code.
6186 * @param pVM Pointer to the VM.
6187 * @param pVCpu Pointer to the VMCPU.
6188 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6189 * out-of-sync. Make sure to update the required fields
6190 * before using them.
6191 *
6192 * @remarks No-long-jmp zone!!!
6193 */
6194DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6195{
6196 HM_DISABLE_PREEMPT_IF_NEEDED();
6197 HMVMX_ASSERT_CPU_SAFE();
6198 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6199 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6200
6201 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6202 and done this from the VMXR0ThreadCtxCallback(). */
6203 if (!pVCpu->hm.s.fLeaveDone)
6204 {
6205 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6206 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6207 pVCpu->hm.s.fLeaveDone = true;
6208 }
6209
6210 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6211 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6212 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6213 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6214 VMMR0ThreadCtxHooksDeregister(pVCpu);
6215
6216 /* Leave HM context. This takes care of local init (term). */
6217 int rc = HMR0LeaveCpu(pVCpu);
6218
6219 HM_RESTORE_PREEMPT_IF_NEEDED();
6220
6221 return rc;
6222}
6223
6224
6225/**
6226 * Does the necessary state syncing before doing a longjmp to ring-3.
6227 *
6228 * @returns VBox status code.
6229 * @param pVM Pointer to the VM.
6230 * @param pVCpu Pointer to the VMCPU.
6231 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6232 * out-of-sync. Make sure to update the required fields
6233 * before using them.
6234 *
6235 * @remarks No-long-jmp zone!!!
6236 */
6237DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6238{
6239 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6240}
6241
6242
6243/**
6244 * Take necessary actions before going back to ring-3.
6245 *
6246 * An action requires us to go back to ring-3. This function does the necessary
6247 * steps before we can safely return to ring-3. This is not the same as longjmps
6248 * to ring-3, this is voluntary and prepares the guest so it may continue
6249 * executing outside HM (recompiler/IEM).
6250 *
6251 * @returns VBox status code.
6252 * @param pVM Pointer to the VM.
6253 * @param pVCpu Pointer to the VMCPU.
6254 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6255 * out-of-sync. Make sure to update the required fields
6256 * before using them.
6257 * @param rcExit The reason for exiting to ring-3. Can be
6258 * VINF_VMM_UNKNOWN_RING3_CALL.
6259 */
6260static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6261{
6262 Assert(pVM);
6263 Assert(pVCpu);
6264 Assert(pMixedCtx);
6265 HMVMX_ASSERT_PREEMPT_SAFE();
6266
6267 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6268 {
6269 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6270 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6271 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6272 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6273 }
6274
6275 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6276 VMMRZCallRing3Disable(pVCpu);
6277 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6278
6279 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6280 if (pVCpu->hm.s.Event.fPending)
6281 {
6282 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6283 Assert(!pVCpu->hm.s.Event.fPending);
6284 }
6285
6286 /* Save guest state and restore host state bits. */
6287 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6288 AssertRCReturn(rc, rc);
6289 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6290
6291 /* Sync recompiler state. */
6292 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6293 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6294 | CPUM_CHANGED_LDTR
6295 | CPUM_CHANGED_GDTR
6296 | CPUM_CHANGED_IDTR
6297 | CPUM_CHANGED_TR
6298 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6299 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6300 if ( pVM->hm.s.fNestedPaging
6301 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6302 {
6303 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6304 }
6305
6306 /*
6307 * Clear the X86_EFL_TF if necessary.
6308 */
6309 if (pVCpu->hm.s.fClearTrapFlag)
6310 {
6311 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6312 pMixedCtx->eflags.Bits.u1TF = 0;
6313 pVCpu->hm.s.fClearTrapFlag = false;
6314 }
6315 /** @todo there seems to be issues with the resume flag when the monitor trap
6316 * flag is pending without being used. Seen early in bios init when
6317 * accessing APIC page in prot mode. */
6318
6319 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6320 if (rcExit != VINF_EM_RAW_INTERRUPT)
6321 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
6322
6323 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6324
6325 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6326 VMMRZCallRing3RemoveNotification(pVCpu);
6327 VMMRZCallRing3Enable(pVCpu);
6328
6329 return rc;
6330}
6331
6332
6333/**
6334 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6335 * longjump to ring-3 and possibly get preempted.
6336 *
6337 * @returns VBox status code.
6338 * @param pVCpu Pointer to the VMCPU.
6339 * @param enmOperation The operation causing the ring-3 longjump.
6340 * @param pvUser Opaque pointer to the guest-CPU context. The data
6341 * may be out-of-sync. Make sure to update the required
6342 * fields before using them.
6343 */
6344DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6345{
6346 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
6347 {
6348 VMMRZCallRing3RemoveNotification(pVCpu);
6349 HM_DISABLE_PREEMPT_IF_NEEDED();
6350
6351 /* If anything here asserts or fails, good luck. */
6352 if (CPUMIsGuestFPUStateActive(pVCpu))
6353 CPUMR0SaveGuestFPU(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6354
6355 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6356
6357#if HC_ARCH_BITS == 64
6358 /* Restore host-state bits that VT-x only restores partially. */
6359 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6360 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6361 {
6362 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6363 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6364 }
6365#endif
6366 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6367 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6368 {
6369 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6370 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6371 }
6372
6373 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6374 VMMR0ThreadCtxHooksDeregister(pVCpu);
6375
6376 HMR0LeaveCpu(pVCpu);
6377 HM_RESTORE_PREEMPT_IF_NEEDED();
6378 return VINF_SUCCESS;
6379 }
6380
6381 Assert(pVCpu);
6382 Assert(pvUser);
6383 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6384 HMVMX_ASSERT_PREEMPT_SAFE();
6385
6386 VMMRZCallRing3Disable(pVCpu);
6387 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6388
6389 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6390 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6391 AssertRCReturn(rc, rc);
6392
6393 VMMRZCallRing3Enable(pVCpu);
6394 return VINF_SUCCESS;
6395}
6396
6397
6398/**
6399 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6400 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6401 *
6402 * @param pVCpu Pointer to the VMCPU.
6403 */
6404DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6405{
6406 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6407 {
6408 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6409 {
6410 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6411 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6412 AssertRC(rc);
6413 }
6414 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6415}
6416
6417
6418/**
6419 * Evaluates the event to be delivered to the guest and sets it as the pending
6420 * event.
6421 *
6422 * @param pVCpu Pointer to the VMCPU.
6423 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6424 * out-of-sync. Make sure to update the required fields
6425 * before using them.
6426 */
6427static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6428{
6429 Assert(!pVCpu->hm.s.Event.fPending);
6430
6431 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6432 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6433 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6434 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6435
6436 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6437 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6438 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6439 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6440 Assert(!TRPMHasTrap(pVCpu));
6441
6442 /** @todo SMI. SMIs take priority over NMIs. */
6443 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6444 {
6445 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6446 if ( !fBlockMovSS
6447 && !fBlockSti)
6448 {
6449 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6450 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6451 uint32_t u32IntrInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6452 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6453
6454 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6455 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6456 }
6457 else
6458 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6459 }
6460 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6461 && !pVCpu->hm.s.fSingleInstruction)
6462 {
6463 /*
6464 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6465 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6466 * evaluated here and not set as pending, solely based on the force-flags.
6467 */
6468 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6469 AssertRC(rc);
6470 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6471 if ( !fBlockInt
6472 && !fBlockSti
6473 && !fBlockMovSS)
6474 {
6475 uint8_t u8Interrupt;
6476 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6477 if (RT_SUCCESS(rc))
6478 {
6479 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6480 uint32_t u32IntrInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6481 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6482
6483 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6484 }
6485 else
6486 {
6487 /** @todo Does this actually happen? If not turn it into an assertion. */
6488 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6489 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6490 }
6491 }
6492 else
6493 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6494 }
6495}
6496
6497
6498/**
6499 * Injects any pending events into the guest if the guest is in a state to
6500 * receive them.
6501 *
6502 * @returns VBox status code (informational status codes included).
6503 * @param pVCpu Pointer to the VMCPU.
6504 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6505 * out-of-sync. Make sure to update the required fields
6506 * before using them.
6507 */
6508static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6509{
6510 HMVMX_ASSERT_PREEMPT_SAFE();
6511 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6512
6513 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6514 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6515 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6516 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6517
6518 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6519 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6520 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6521 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6522 Assert(!TRPMHasTrap(pVCpu));
6523
6524 int rc = VINF_SUCCESS;
6525 if (pVCpu->hm.s.Event.fPending)
6526 {
6527#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6528 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6529 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6530 {
6531 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6532 AssertRCReturn(rc, rc);
6533 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6534 Assert(!fBlockInt);
6535 Assert(!fBlockSti);
6536 Assert(!fBlockMovSS);
6537 }
6538 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6539 {
6540 Assert(!fBlockSti);
6541 Assert(!fBlockMovSS);
6542 }
6543#endif
6544 Log4(("Injecting pending event vcpu[%RU32] u64IntrInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo));
6545 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6546 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6547 AssertRCReturn(rc, rc);
6548
6549 /* Update the interruptibility-state as it could have been changed by
6550 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6551 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6552 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6553
6554#ifdef VBOX_WITH_STATISTICS
6555 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6556 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6557 else
6558 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6559#endif
6560 }
6561
6562 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6563 int rc2 = VINF_SUCCESS;
6564 if ( fBlockSti
6565 || fBlockMovSS)
6566 {
6567 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6568 {
6569 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6570 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6571 {
6572 /*
6573 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6574 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6575 * See Intel spec. 27.3.4 "Saving Non-Register State".
6576 */
6577 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6578 AssertRCReturn(rc, rc);
6579 }
6580 }
6581 else
6582 {
6583 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6584 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6585 uIntrState = 0;
6586 }
6587 }
6588
6589 /*
6590 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6591 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6592 */
6593 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6594 AssertRC(rc2);
6595
6596 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6597 return rc;
6598}
6599
6600
6601/**
6602 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6603 *
6604 * @param pVCpu Pointer to the VMCPU.
6605 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6606 * out-of-sync. Make sure to update the required fields
6607 * before using them.
6608 */
6609DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6610{
6611 uint32_t u32IntrInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6612 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6613}
6614
6615
6616/**
6617 * Injects a double-fault (#DF) exception into the VM.
6618 *
6619 * @returns VBox status code (informational status code included).
6620 * @param pVCpu Pointer to the VMCPU.
6621 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6622 * out-of-sync. Make sure to update the required fields
6623 * before using them.
6624 */
6625DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6626{
6627 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6628 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6629 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6630 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6631 puIntrState);
6632}
6633
6634
6635/**
6636 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6637 *
6638 * @param pVCpu Pointer to the VMCPU.
6639 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6640 * out-of-sync. Make sure to update the required fields
6641 * before using them.
6642 */
6643DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6644{
6645 uint32_t u32IntrInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6646 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6647 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6648}
6649
6650
6651/**
6652 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6653 *
6654 * @param pVCpu Pointer to the VMCPU.
6655 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6656 * out-of-sync. Make sure to update the required fields
6657 * before using them.
6658 * @param cbInstr The value of RIP that is to be pushed on the guest
6659 * stack.
6660 */
6661DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6662{
6663 uint32_t u32IntrInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6664 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6665 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6666}
6667
6668
6669/**
6670 * Injects a general-protection (#GP) fault into the VM.
6671 *
6672 * @returns VBox status code (informational status code included).
6673 * @param pVCpu Pointer to the VMCPU.
6674 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6675 * out-of-sync. Make sure to update the required fields
6676 * before using them.
6677 * @param u32ErrorCode The error code associated with the #GP.
6678 */
6679DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6680 uint32_t *puIntrState)
6681{
6682 uint32_t u32IntrInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6683 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6684 if (fErrorCodeValid)
6685 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6686 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6687 puIntrState);
6688}
6689
6690
6691/**
6692 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6693 *
6694 * @param pVCpu Pointer to the VMCPU.
6695 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6696 * out-of-sync. Make sure to update the required fields
6697 * before using them.
6698 * @param uVector The software interrupt vector number.
6699 * @param cbInstr The value of RIP that is to be pushed on the guest
6700 * stack.
6701 */
6702DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6703{
6704 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6705 if ( uVector == X86_XCPT_BP
6706 || uVector == X86_XCPT_OF)
6707 {
6708 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6709 }
6710 else
6711 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6712 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6713}
6714
6715
6716/**
6717 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6718 * stack.
6719 *
6720 * @returns VBox status code (information status code included).
6721 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6722 * @param pVM Pointer to the VM.
6723 * @param pMixedCtx Pointer to the guest-CPU context.
6724 * @param uValue The value to push to the guest stack.
6725 */
6726DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6727{
6728 /*
6729 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6730 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6731 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6732 */
6733 if (pMixedCtx->sp == 1)
6734 return VINF_EM_RESET;
6735 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6736 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6737 AssertRCReturn(rc, rc);
6738 return rc;
6739}
6740
6741
6742/**
6743 * Injects an event into the guest upon VM-entry by updating the relevant fields
6744 * in the VM-entry area in the VMCS.
6745 *
6746 * @returns VBox status code (informational error codes included).
6747 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6748 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6749 *
6750 * @param pVCpu Pointer to the VMCPU.
6751 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6752 * be out-of-sync. Make sure to update the required
6753 * fields before using them.
6754 * @param u64IntrInfo The VM-entry interruption-information field.
6755 * @param cbInstr The VM-entry instruction length in bytes (for
6756 * software interrupts, exceptions and privileged
6757 * software exceptions).
6758 * @param u32ErrCode The VM-entry exception error code.
6759 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6760 * @param puIntrState Pointer to the current guest interruptibility-state.
6761 * This interruptibility-state will be updated if
6762 * necessary. This cannot not be NULL.
6763 *
6764 * @remarks Requires CR0!
6765 * @remarks No-long-jump zone!!!
6766 */
6767static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6768 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6769{
6770 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6771 AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6772 Assert(puIntrState);
6773 uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6774
6775 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6776 const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6777
6778#ifdef VBOX_STRICT
6779 /* Validate the error-code-valid bit for hardware exceptions. */
6780 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6781 {
6782 switch (uVector)
6783 {
6784 case X86_XCPT_PF:
6785 case X86_XCPT_DF:
6786 case X86_XCPT_TS:
6787 case X86_XCPT_NP:
6788 case X86_XCPT_SS:
6789 case X86_XCPT_GP:
6790 case X86_XCPT_AC:
6791 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6792 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6793 /* fallthru */
6794 default:
6795 break;
6796 }
6797 }
6798#endif
6799
6800 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
6801 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6802 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6803
6804 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6805
6806 /* We require CR0 to check if the guest is in real-mode. */
6807 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6808 AssertRCReturn(rc, rc);
6809
6810 /*
6811 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6812 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6813 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6814 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6815 */
6816 if (CPUMIsGuestInRealModeEx(pMixedCtx))
6817 {
6818 PVM pVM = pVCpu->CTX_SUFF(pVM);
6819 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6820 {
6821 Assert(PDMVmmDevHeapIsEnabled(pVM));
6822 Assert(pVM->hm.s.vmx.pRealModeTSS);
6823
6824 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6825 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6826 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6827 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6828 AssertRCReturn(rc, rc);
6829 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6830
6831 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6832 const size_t cbIdtEntry = sizeof(X86IDTR16);
6833 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6834 {
6835 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6836 if (uVector == X86_XCPT_DF)
6837 return VINF_EM_RESET;
6838 else if (uVector == X86_XCPT_GP)
6839 {
6840 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6841 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6842 }
6843
6844 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6845 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6846 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
6847 }
6848
6849 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6850 uint16_t uGuestIp = pMixedCtx->ip;
6851 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6852 {
6853 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
6854 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
6855 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6856 }
6857 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6858 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6859
6860 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6861 X86IDTR16 IdtEntry;
6862 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6863 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6864 AssertRCReturn(rc, rc);
6865
6866 /* Construct the stack frame for the interrupt/exception handler. */
6867 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6868 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6869 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6870 AssertRCReturn(rc, rc);
6871
6872 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
6873 if (rc == VINF_SUCCESS)
6874 {
6875 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
6876 pMixedCtx->rip = IdtEntry.offSel;
6877 pMixedCtx->cs.Sel = IdtEntry.uSel;
6878 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6879 if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6880 && uVector == X86_XCPT_PF)
6881 {
6882 pMixedCtx->cr2 = GCPtrFaultAddress;
6883 }
6884
6885 /* If any other guest-state bits are changed here, make sure to update
6886 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6887 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS
6888 | HM_CHANGED_GUEST_RIP
6889 | HM_CHANGED_GUEST_RFLAGS
6890 | HM_CHANGED_GUEST_RSP;
6891
6892 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6893 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6894 {
6895 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6896 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6897 Log4(("Clearing inhibition due to STI.\n"));
6898 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6899 }
6900 Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6901
6902 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
6903 it, if we are returning to ring-3 before executing guest code. */
6904 pVCpu->hm.s.Event.fPending = false;
6905 }
6906 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6907 return rc;
6908 }
6909 else
6910 {
6911 /*
6912 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6913 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6914 */
6915 u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6916 }
6917 }
6918
6919 /* Validate. */
6920 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6921 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6922 Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6923
6924 /* Inject. */
6925 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6926 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6927 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6928 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6929
6930 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6931 && uVector == X86_XCPT_PF)
6932 {
6933 pMixedCtx->cr2 = GCPtrFaultAddress;
6934 }
6935
6936 Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6937 u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6938
6939 AssertRCReturn(rc, rc);
6940 return rc;
6941}
6942
6943
6944/**
6945 * Clears the interrupt-window exiting control in the VMCS and if necessary
6946 * clears the current event in the VMCS as well.
6947 *
6948 * @returns VBox status code.
6949 * @param pVCpu Pointer to the VMCPU.
6950 *
6951 * @remarks Use this function only to clear events that have not yet been
6952 * delivered to the guest but are injected in the VMCS!
6953 * @remarks No-long-jump zone!!!
6954 */
6955static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
6956{
6957 int rc;
6958 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
6959
6960 /* Clear interrupt-window exiting control. */
6961 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
6962 {
6963 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6964 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6965 AssertRC(rc);
6966 }
6967
6968 if (!pVCpu->hm.s.Event.fPending)
6969 return;
6970
6971#ifdef VBOX_STRICT
6972 uint32_t u32EntryInfo;
6973 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
6974 AssertRC(rc);
6975 Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
6976#endif
6977
6978 /* Clear the entry-interruption field (including the valid bit). */
6979 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
6980 AssertRC(rc);
6981
6982 /* Clear the pending debug exception field. */
6983 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
6984 AssertRC(rc);
6985}
6986
6987
6988/**
6989 * Enters the VT-x session.
6990 *
6991 * @returns VBox status code.
6992 * @param pVM Pointer to the VM.
6993 * @param pVCpu Pointer to the VMCPU.
6994 * @param pCpu Pointer to the CPU info struct.
6995 */
6996VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
6997{
6998 AssertPtr(pVM);
6999 AssertPtr(pVCpu);
7000 Assert(pVM->hm.s.vmx.fSupported);
7001 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7002 NOREF(pCpu);
7003
7004 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7005 Assert((pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE))
7006 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7007
7008#ifdef VBOX_STRICT
7009 /* Make sure we're in VMX root mode. */
7010 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7011 if (!(u32HostCR4 & X86_CR4_VMXE))
7012 {
7013 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7014 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7015 }
7016#endif
7017
7018 /*
7019 * Load the VCPU's VMCS as the current (and active) one.
7020 */
7021 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7022 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7023 if (RT_FAILURE(rc))
7024 return rc;
7025
7026 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7027 pVCpu->hm.s.fLeaveDone = false;
7028 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7029
7030 return VINF_SUCCESS;
7031}
7032
7033
7034/**
7035 * The thread-context callback (only on platforms which support it).
7036 *
7037 * @param enmEvent The thread-context event.
7038 * @param pVCpu Pointer to the VMCPU.
7039 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7040 * @thread EMT.
7041 */
7042VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7043{
7044 switch (enmEvent)
7045 {
7046 case RTTHREADCTXEVENT_PREEMPTING:
7047 {
7048 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7049 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7050 VMCPU_ASSERT_EMT(pVCpu);
7051
7052 PVM pVM = pVCpu->CTX_SUFF(pVM);
7053 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7054
7055 /* No longjmps (logger flushes, locks) in this fragile context. */
7056 VMMRZCallRing3Disable(pVCpu);
7057 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7058
7059 /*
7060 * Restore host-state (FPU, debug etc.)
7061 */
7062 if (!pVCpu->hm.s.fLeaveDone)
7063 {
7064 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7065 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7066 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7067 pVCpu->hm.s.fLeaveDone = true;
7068 }
7069
7070 /* Leave HM context, takes care of local init (term). */
7071 int rc = HMR0LeaveCpu(pVCpu);
7072 AssertRC(rc); NOREF(rc);
7073
7074 /* Restore longjmp state. */
7075 VMMRZCallRing3Enable(pVCpu);
7076 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7077 break;
7078 }
7079
7080 case RTTHREADCTXEVENT_RESUMED:
7081 {
7082 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7083 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7084 VMCPU_ASSERT_EMT(pVCpu);
7085
7086 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7087 VMMRZCallRing3Disable(pVCpu);
7088 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7089
7090 /* Initialize the bare minimum state required for HM. This takes care of
7091 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7092 int rc = HMR0EnterCpu(pVCpu);
7093 AssertRC(rc);
7094 Assert((pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE))
7095 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7096
7097 /* Load the active VMCS as the current one. */
7098 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7099 {
7100 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7101 AssertRC(rc); NOREF(rc);
7102 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7103 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7104 }
7105 pVCpu->hm.s.fLeaveDone = false;
7106
7107 /* Restore longjmp state. */
7108 VMMRZCallRing3Enable(pVCpu);
7109 break;
7110 }
7111
7112 default:
7113 break;
7114 }
7115}
7116
7117
7118/**
7119 * Saves the host state in the VMCS host-state.
7120 * Sets up the VM-exit MSR-load area.
7121 *
7122 * The CPU state will be loaded from these fields on every successful VM-exit.
7123 *
7124 * @returns VBox status code.
7125 * @param pVM Pointer to the VM.
7126 * @param pVCpu Pointer to the VMCPU.
7127 *
7128 * @remarks No-long-jump zone!!!
7129 */
7130static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7131{
7132 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7133
7134 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
7135 return VINF_SUCCESS;
7136
7137 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7138 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7139
7140 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7141 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7142
7143 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7144 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7145
7146 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
7147 return rc;
7148}
7149
7150
7151/**
7152 * Saves the host state in the VMCS host-state.
7153 *
7154 * @returns VBox status code.
7155 * @param pVM Pointer to the VM.
7156 * @param pVCpu Pointer to the VMCPU.
7157 *
7158 * @remarks No-long-jump zone!!!
7159 */
7160VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7161{
7162 AssertPtr(pVM);
7163 AssertPtr(pVCpu);
7164
7165 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7166
7167 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7168 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7169 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7170 return hmR0VmxSaveHostState(pVM, pVCpu);
7171}
7172
7173
7174/**
7175 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7176 * loaded from these fields on every successful VM-entry.
7177 *
7178 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7179 * Sets up the VM-entry controls.
7180 * Sets up the appropriate VMX non-root function to execute guest code based on
7181 * the guest CPU mode.
7182 *
7183 * @returns VBox status code.
7184 * @param pVM Pointer to the VM.
7185 * @param pVCpu Pointer to the VMCPU.
7186 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7187 * out-of-sync. Make sure to update the required fields
7188 * before using them.
7189 *
7190 * @remarks No-long-jump zone!!!
7191 */
7192static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7193{
7194 AssertPtr(pVM);
7195 AssertPtr(pVCpu);
7196 AssertPtr(pMixedCtx);
7197 HMVMX_ASSERT_PREEMPT_SAFE();
7198
7199#ifdef LOG_ENABLED
7200 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7201 * probably not initialized yet? Anyway this will do for now.
7202 *
7203 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7204 * interface and disable ring-3 calls when thread-context hooks are not
7205 * available. */
7206 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7207 VMMR0LogFlushDisable(pVCpu);
7208#endif
7209
7210 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7211
7212 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7213
7214 /* Determine real-on-v86 mode. */
7215 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7216 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7217 && CPUMIsGuestInRealModeEx(pMixedCtx))
7218 {
7219 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7220 }
7221
7222 /*
7223 * Load the guest-state into the VMCS.
7224 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7225 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7226 */
7227 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7228 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7229
7230 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7231 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7232 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7233
7234 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7235 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7236 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7237
7238 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7239 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7240
7241 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7242 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7243
7244 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7245 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7246 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7247
7248 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7249 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7250
7251 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7252 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7253
7254 /*
7255 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7256 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7257 */
7258 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7259 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7260
7261 /* Clear any unused and reserved bits. */
7262 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
7263
7264#ifdef LOG_ENABLED
7265 /* Only reenable log-flushing if the caller has it enabled. */
7266 if (!fCallerDisabledLogFlush)
7267 VMMR0LogFlushEnable(pVCpu);
7268#endif
7269
7270 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7271 return rc;
7272}
7273
7274
7275/**
7276 * Loads the state shared between the host and guest into the VMCS.
7277 *
7278 * @param pVM Pointer to the VM.
7279 * @param pVCpu Pointer to the VMCPU.
7280 * @param pCtx Pointer to the guest-CPU context.
7281 *
7282 * @remarks No-long-jump zone!!!
7283 */
7284static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7285{
7286 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7287 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7288
7289 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
7290 {
7291 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7292 AssertRC(rc);
7293 }
7294
7295 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
7296 {
7297 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7298 AssertRC(rc);
7299
7300 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7301 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
7302 {
7303 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7304 AssertRC(rc);
7305 }
7306 }
7307
7308 AssertMsg(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE), ("fContextUseFlags=%#x\n",
7309 pVCpu->hm.s.fContextUseFlags));
7310}
7311
7312
7313/**
7314 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7315 *
7316 * @param pVM Pointer to the VM.
7317 * @param pVCpu Pointer to the VMCPU.
7318 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7319 * out-of-sync. Make sure to update the required fields
7320 * before using them.
7321 */
7322DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7323{
7324 HMVMX_ASSERT_PREEMPT_SAFE();
7325
7326 Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7327#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7328 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
7329#endif
7330
7331 if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7332 {
7333 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7334 AssertRC(rc);
7335 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7336 }
7337 else if (pVCpu->hm.s.fContextUseFlags)
7338 {
7339 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7340 AssertRC(rc);
7341 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7342 }
7343
7344 /* All the guest state bits should be loaded except maybe the host context and shared host/guest bits. */
7345 AssertMsg( !(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST)
7346 || !(pVCpu->hm.s.fContextUseFlags & ~(HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE)),
7347 ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7348
7349#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7350 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7351 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7352 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7353#endif
7354}
7355
7356
7357/**
7358 * Does the preparations before executing guest code in VT-x.
7359 *
7360 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7361 * recompiler. We must be cautious what we do here regarding committing
7362 * guest-state information into the VMCS assuming we assuredly execute the
7363 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7364 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7365 * so that the recompiler can (and should) use them when it resumes guest
7366 * execution. Otherwise such operations must be done when we can no longer
7367 * exit to ring-3.
7368 *
7369 * @returns Strict VBox status code.
7370 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7371 * have been disabled.
7372 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7373 * double-fault into the guest.
7374 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7375 *
7376 * @param pVM Pointer to the VM.
7377 * @param pVCpu Pointer to the VMCPU.
7378 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7379 * out-of-sync. Make sure to update the required fields
7380 * before using them.
7381 * @param pVmxTransient Pointer to the VMX transient structure.
7382 *
7383 * @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7384 * interrupts will be disabled.
7385 */
7386static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7387{
7388 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7389
7390#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7391 PGMRZDynMapFlushAutoSet(pVCpu);
7392#endif
7393
7394 /* Check force flag actions that might require us to go back to ring-3. */
7395 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7396 if (rc != VINF_SUCCESS)
7397 return rc;
7398
7399#ifndef IEM_VERIFICATION_MODE_FULL
7400 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7401 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7402 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7403 {
7404 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7405 RTGCPHYS GCPhysApicBase;
7406 GCPhysApicBase = pMixedCtx->msrApicBase;
7407 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7408
7409 /* Unalias any existing mapping. */
7410 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7411 AssertRCReturn(rc, rc);
7412
7413 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7414 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7415 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7416 AssertRCReturn(rc, rc);
7417
7418 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7419 }
7420#endif /* !IEM_VERIFICATION_MODE_FULL */
7421
7422 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7423 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7424
7425 /*
7426 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7427 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7428 */
7429 if (TRPMHasTrap(pVCpu))
7430 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7431 else if (!pVCpu->hm.s.Event.fPending)
7432 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7433
7434 /*
7435 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7436 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7437 */
7438 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7439 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7440 {
7441 Assert(rc == VINF_EM_RESET);
7442 return rc;
7443 }
7444
7445 /*
7446 * No longjmps to ring-3 from this point on!!!
7447 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7448 * This also disables flushing of the R0-logger instance (if any).
7449 */
7450 VMMRZCallRing3Disable(pVCpu);
7451
7452 /*
7453 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7454 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7455 *
7456 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7457 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7458 *
7459 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7460 * executing guest code.
7461 */
7462 pVmxTransient->uEflags = ASMIntDisableFlags();
7463 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7464 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7465 {
7466 hmR0VmxClearEventVmcs(pVCpu);
7467 ASMSetFlags(pVmxTransient->uEflags);
7468 VMMRZCallRing3Enable(pVCpu);
7469 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7470 return VINF_EM_RAW_TO_R3;
7471 }
7472 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7473 {
7474 hmR0VmxClearEventVmcs(pVCpu);
7475 ASMSetFlags(pVmxTransient->uEflags);
7476 VMMRZCallRing3Enable(pVCpu);
7477 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7478 return VINF_EM_RAW_INTERRUPT;
7479 }
7480
7481 /* We've injected any pending events. This is really the point of no return (to ring-3). */
7482 pVCpu->hm.s.Event.fPending = false;
7483
7484 return VINF_SUCCESS;
7485}
7486
7487
7488/**
7489 * Prepares to run guest code in VT-x and we've committed to doing so. This
7490 * means there is no backing out to ring-3 or anywhere else at this
7491 * point.
7492 *
7493 * @param pVM Pointer to the VM.
7494 * @param pVCpu Pointer to the VMCPU.
7495 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7496 * out-of-sync. Make sure to update the required fields
7497 * before using them.
7498 * @param pVmxTransient Pointer to the VMX transient structure.
7499 *
7500 * @remarks Called with preemption disabled.
7501 * @remarks No-long-jump zone!!!
7502 */
7503static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7504{
7505 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7506 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7507 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7508
7509 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7510 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7511
7512 /*
7513 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7514 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7515 * Reload only the necessary state, the assertion will catch if other parts of the code
7516 * change.
7517 */
7518 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7519 {
7520 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7521 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7522 }
7523
7524 /*
7525 * Load the host state bits as we may've been preempted (only happens when
7526 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
7527 */
7528 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
7529 {
7530 /* This ASSUMES that pfnStartVM has been set up already. */
7531 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7532 AssertRC(rc);
7533 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7534 }
7535 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7536
7537#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7538 if (!CPUMIsGuestFPUStateActive(pVCpu))
7539 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
7540 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
7541#endif
7542
7543 /*
7544 * Load the state shared between host and guest (FPU, debug).
7545 */
7546 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE)
7547 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7548 AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7549
7550 /*
7551 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7552 */
7553 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7554 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7555
7556 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7557 RTCPUID idCurrentCpu = pCpu->idCpu;
7558 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7559 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
7560 {
7561 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7562 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7563 }
7564
7565 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7566 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7567 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7568 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7569
7570 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7571
7572 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7573 to start executing. */
7574
7575#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7576 /*
7577 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7578 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7579 */
7580 if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7581 && !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7582 {
7583 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7584 uint64_t u64HostTscAux = 0;
7585 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7586 AssertRC(rc2);
7587 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7588 }
7589#endif
7590}
7591
7592
7593/**
7594 * Performs some essential restoration of state after running guest code in
7595 * VT-x.
7596 *
7597 * @param pVM Pointer to the VM.
7598 * @param pVCpu Pointer to the VMCPU.
7599 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7600 * out-of-sync. Make sure to update the required fields
7601 * before using them.
7602 * @param pVmxTransient Pointer to the VMX transient structure.
7603 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7604 *
7605 * @remarks Called with interrupts disabled, and returns with interrups enabled!
7606 *
7607 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7608 * unconditionally when it is safe to do so.
7609 */
7610static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7611{
7612 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7613
7614 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7615 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7616 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7617 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7618 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7619
7620 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7621 {
7622#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7623 /* Restore host's TSC_AUX. */
7624 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7625 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7626#endif
7627 /** @todo Find a way to fix hardcoding a guestimate. */
7628 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7629 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7630 }
7631
7632 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7633 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7634 Assert(!(ASMGetFlags() & X86_EFL_IF));
7635 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7636
7637#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
7638 if (CPUMIsGuestFPUStateActive(pVCpu))
7639 {
7640 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7641 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7642 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
7643 }
7644#endif
7645
7646 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7647 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7648 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7649 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7650
7651 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7652 uint32_t uExitReason;
7653 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7654 rc |= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7655 AssertRC(rc);
7656 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7657 pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7658
7659 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7660 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7661 {
7662 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7663 pVmxTransient->fVMEntryFailed));
7664 return;
7665 }
7666
7667 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7668 {
7669 /* Update the guest interruptibility-state from the VMCS. */
7670 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7671#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7672 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7673 AssertRC(rc);
7674#endif
7675 /*
7676 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7677 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7678 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7679 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7680 */
7681 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7682 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7683 {
7684 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7685 AssertRC(rc);
7686 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
7687 }
7688 }
7689}
7690
7691
7692
7693/**
7694 * Runs the guest code using VT-x the normal way.
7695 *
7696 * @returns VBox status code.
7697 * @param pVM Pointer to the VM.
7698 * @param pVCpu Pointer to the VMCPU.
7699 * @param pCtx Pointer to the guest-CPU context.
7700 *
7701 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
7702 * @remarks Called with preemption disabled.
7703 */
7704static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7705{
7706 VMXTRANSIENT VmxTransient;
7707 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7708 int rc = VERR_INTERNAL_ERROR_5;
7709 uint32_t cLoops = 0;
7710
7711 for (;; cLoops++)
7712 {
7713 Assert(!HMR0SuspendPending());
7714 HMVMX_ASSERT_CPU_SAFE();
7715
7716 /* Preparatory work for running guest code, this may force us to return
7717 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7718 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7719 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7720 if (rc != VINF_SUCCESS)
7721 break;
7722
7723 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7724 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7725 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7726
7727 /* Restore any residual host-state and save any bits shared between host
7728 and guest into the guest-CPU state. Re-enables interrupts! */
7729 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7730
7731 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7732 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7733 {
7734 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7735 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7736 return rc;
7737 }
7738
7739 /* Handle the VM-exit. */
7740 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7741 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7742 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7743 HMVMX_START_EXIT_DISPATCH_PROF();
7744#ifdef HMVMX_USE_FUNCTION_TABLE
7745 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7746#else
7747 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7748#endif
7749 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7750 if (rc != VINF_SUCCESS)
7751 break;
7752 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7753 {
7754 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7755 rc = VINF_EM_RAW_INTERRUPT;
7756 break;
7757 }
7758 }
7759
7760 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7761 return rc;
7762}
7763
7764
7765/**
7766 * Single steps guest code using VT-x.
7767 *
7768 * @returns VBox status code.
7769 * @param pVM Pointer to the VM.
7770 * @param pVCpu Pointer to the VMCPU.
7771 * @param pCtx Pointer to the guest-CPU context.
7772 *
7773 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7774 * @remarks Called with preemption disabled.
7775 */
7776static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7777{
7778 VMXTRANSIENT VmxTransient;
7779 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7780 int rc = VERR_INTERNAL_ERROR_5;
7781 uint32_t cLoops = 0;
7782 uint16_t uCsStart = pCtx->cs.Sel;
7783 uint64_t uRipStart = pCtx->rip;
7784
7785 for (;; cLoops++)
7786 {
7787 Assert(!HMR0SuspendPending());
7788 HMVMX_ASSERT_CPU_SAFE();
7789
7790 /* Preparatory work for running guest code, this may force us to return
7791 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7792 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7793 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7794 if (rc != VINF_SUCCESS)
7795 break;
7796
7797 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7798 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7799 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7800
7801 /* Restore any residual host-state and save any bits shared between host
7802 and guest into the guest-CPU state. Re-enables interrupts! */
7803 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7804
7805 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7806 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7807 {
7808 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7809 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7810 return rc;
7811 }
7812
7813 /* Handle the VM-exit. */
7814 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7815 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7816 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7817 HMVMX_START_EXIT_DISPATCH_PROF();
7818#ifdef HMVMX_USE_FUNCTION_TABLE
7819 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7820#else
7821 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7822#endif
7823 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7824 if (rc != VINF_SUCCESS)
7825 break;
7826 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7827 {
7828 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7829 rc = VINF_EM_RAW_INTERRUPT;
7830 break;
7831 }
7832
7833 /*
7834 * Did the RIP change, if so, consider it a single step.
7835 * Otherwise, make sure one of the TFs gets set.
7836 */
7837 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7838 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7839 AssertRCReturn(rc2, rc2);
7840 if ( pCtx->rip != uRipStart
7841 || pCtx->cs.Sel != uCsStart)
7842 {
7843 rc = VINF_EM_DBG_STEPPED;
7844 break;
7845 }
7846 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
7847 }
7848
7849 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7850 return rc;
7851}
7852
7853
7854/**
7855 * Runs the guest code using VT-x.
7856 *
7857 * @returns VBox status code.
7858 * @param pVM Pointer to the VM.
7859 * @param pVCpu Pointer to the VMCPU.
7860 * @param pCtx Pointer to the guest-CPU context.
7861 *
7862 * @remarks Called with preemption disabled.
7863 */
7864VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7865{
7866 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7867 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
7868 HMVMX_ASSERT_PREEMPT_SAFE();
7869
7870 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
7871
7872 int rc;
7873 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7874 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7875 else
7876 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7877
7878 if (rc == VERR_EM_INTERPRETER)
7879 rc = VINF_EM_RAW_EMULATE_INSTR;
7880 else if (rc == VINF_EM_RESET)
7881 rc = VINF_EM_TRIPLE_FAULT;
7882
7883 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7884 if (RT_FAILURE(rc2))
7885 {
7886 pVCpu->hm.s.u32HMError = rc;
7887 rc = rc2;
7888 }
7889 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
7890 return rc;
7891}
7892
7893
7894#ifndef HMVMX_USE_FUNCTION_TABLE
7895DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7896{
7897 int rc;
7898 switch (rcReason)
7899 {
7900 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7901 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7902 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7903 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7904 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7905 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7906 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7907 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7908 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7909 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7910 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7911 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7912 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7913 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7914 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7915 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7916 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7917 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7918 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7919 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7920 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7921 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7922 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7923 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7924 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7925 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7926 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7927 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7928 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7929 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7930 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7931 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7932 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7933
7934 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7935 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7936 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7937 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7938 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7939 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7940 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7941 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7942 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7943
7944 case VMX_EXIT_VMCALL:
7945 case VMX_EXIT_VMCLEAR:
7946 case VMX_EXIT_VMLAUNCH:
7947 case VMX_EXIT_VMPTRLD:
7948 case VMX_EXIT_VMPTRST:
7949 case VMX_EXIT_VMREAD:
7950 case VMX_EXIT_VMRESUME:
7951 case VMX_EXIT_VMWRITE:
7952 case VMX_EXIT_VMXOFF:
7953 case VMX_EXIT_VMXON:
7954 case VMX_EXIT_INVEPT:
7955 case VMX_EXIT_INVVPID:
7956 case VMX_EXIT_VMFUNC:
7957 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7958 break;
7959 default:
7960 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7961 break;
7962 }
7963 return rc;
7964}
7965#endif
7966
7967#ifdef DEBUG
7968/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7969# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7970 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7971
7972# define HMVMX_ASSERT_PREEMPT_CPUID() \
7973 do \
7974 { \
7975 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7976 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7977 } while (0)
7978
7979# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7980 do { \
7981 AssertPtr(pVCpu); \
7982 AssertPtr(pMixedCtx); \
7983 AssertPtr(pVmxTransient); \
7984 Assert(pVmxTransient->fVMEntryFailed == false); \
7985 Assert(ASMIntAreEnabled()); \
7986 HMVMX_ASSERT_PREEMPT_SAFE(); \
7987 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7988 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7989 HMVMX_ASSERT_PREEMPT_SAFE(); \
7990 if (VMMR0IsLogFlushDisabled(pVCpu)) \
7991 HMVMX_ASSERT_PREEMPT_CPUID(); \
7992 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7993 } while (0)
7994
7995# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7996 do { \
7997 Log4Func(("\n")); \
7998 } while(0)
7999#else /* Release builds */
8000# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
8001# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
8002#endif
8003
8004
8005/**
8006 * Advances the guest RIP after reading it from the VMCS.
8007 *
8008 * @returns VBox status code.
8009 * @param pVCpu Pointer to the VMCPU.
8010 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8011 * out-of-sync. Make sure to update the required fields
8012 * before using them.
8013 * @param pVmxTransient Pointer to the VMX transient structure.
8014 *
8015 * @remarks No-long-jump zone!!!
8016 */
8017DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8018{
8019 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8020 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8021 AssertRCReturn(rc, rc);
8022
8023 pMixedCtx->rip += pVmxTransient->cbInstr;
8024 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
8025 return rc;
8026}
8027
8028
8029/**
8030 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8031 * and update error record fields accordingly.
8032 *
8033 * @return VMX_IGS_* return codes.
8034 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8035 * wrong with the guest state.
8036 *
8037 * @param pVM Pointer to the VM.
8038 * @param pVCpu Pointer to the VMCPU.
8039 * @param pCtx Pointer to the guest-CPU state.
8040 */
8041static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8042{
8043#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8044#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8045 uError = (err); \
8046 break; \
8047 } else do {} while (0)
8048/* Duplicate of IEM_IS_CANONICAL(). */
8049#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8050
8051 int rc;
8052 uint32_t uError = VMX_IGS_ERROR;
8053 uint32_t u32Val;
8054 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8055
8056 do
8057 {
8058 /*
8059 * CR0.
8060 */
8061 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8062 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8063 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8064 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8065 if (fUnrestrictedGuest)
8066 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8067
8068 uint32_t u32GuestCR0;
8069 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8070 AssertRCBreak(rc);
8071 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8072 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8073 if ( !fUnrestrictedGuest
8074 && (u32GuestCR0 & X86_CR0_PG)
8075 && !(u32GuestCR0 & X86_CR0_PE))
8076 {
8077 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8078 }
8079
8080 /*
8081 * CR4.
8082 */
8083 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8084 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8085
8086 uint32_t u32GuestCR4;
8087 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
8088 AssertRCBreak(rc);
8089 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8090 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8091
8092 /*
8093 * IA32_DEBUGCTL MSR.
8094 */
8095 uint64_t u64Val;
8096 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8097 AssertRCBreak(rc);
8098 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8099 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8100 {
8101 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8102 }
8103 uint64_t u64DebugCtlMsr = u64Val;
8104
8105#ifdef VBOX_STRICT
8106 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
8107 AssertRCBreak(rc);
8108 Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8109#endif
8110 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8111
8112 /*
8113 * RIP and RFLAGS.
8114 */
8115 uint32_t u32Eflags;
8116#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8117 if (HMVMX_IS_64BIT_HOST_MODE())
8118 {
8119 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8120 AssertRCBreak(rc);
8121 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8122 if ( !fLongModeGuest
8123 || !pCtx->cs.Attr.n.u1Long)
8124 {
8125 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8126 }
8127 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8128 * must be identical if the "IA32e mode guest" VM-entry control is 1
8129 * and CS.L is 1. No check applies if the CPU supports 64
8130 * linear-address bits. */
8131
8132 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8133 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8134 AssertRCBreak(rc);
8135 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8136 VMX_IGS_RFLAGS_RESERVED);
8137 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8138 u32Eflags = u64Val;
8139 }
8140 else
8141#endif
8142 {
8143 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8144 AssertRCBreak(rc);
8145 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8146 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8147 }
8148
8149 if ( fLongModeGuest
8150 || ( fUnrestrictedGuest
8151 && !(u32GuestCR0 & X86_CR0_PE)))
8152 {
8153 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8154 }
8155
8156 uint32_t u32EntryInfo;
8157 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8158 AssertRCBreak(rc);
8159 if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8160 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8161 {
8162 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8163 }
8164
8165 /*
8166 * 64-bit checks.
8167 */
8168#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8169 if (HMVMX_IS_64BIT_HOST_MODE())
8170 {
8171 if ( fLongModeGuest
8172 && !fUnrestrictedGuest)
8173 {
8174 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
8175 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
8176 }
8177
8178 if ( !fLongModeGuest
8179 && (u32GuestCR4 & X86_CR4_PCIDE))
8180 {
8181 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8182 }
8183
8184 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8185 * 51:32 beyond the processor's physical-address width are 0. */
8186
8187 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8188 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8189 {
8190 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8191 }
8192
8193 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8194 AssertRCBreak(rc);
8195 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8196
8197 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8198 AssertRCBreak(rc);
8199 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8200 }
8201#endif
8202
8203 /*
8204 * PERF_GLOBAL MSR.
8205 */
8206 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8207 {
8208 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8209 AssertRCBreak(rc);
8210 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8211 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8212 }
8213
8214 /*
8215 * PAT MSR.
8216 */
8217 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8218 {
8219 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8220 AssertRCBreak(rc);
8221 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8222 for (unsigned i = 0; i < 8; i++)
8223 {
8224 uint8_t u8Val = (u64Val & 0x7);
8225 if ( u8Val != 0 /* UC */
8226 || u8Val != 1 /* WC */
8227 || u8Val != 4 /* WT */
8228 || u8Val != 5 /* WP */
8229 || u8Val != 6 /* WB */
8230 || u8Val != 7 /* UC- */)
8231 {
8232 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8233 }
8234 u64Val >>= 3;
8235 }
8236 }
8237
8238 /*
8239 * EFER MSR.
8240 */
8241 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8242 {
8243 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8244 AssertRCBreak(rc);
8245 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8246 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8247 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8248 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8249 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8250 || (u64Val & MSR_K6_EFER_LMA) == (u32GuestCR0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8251 }
8252
8253 /*
8254 * Segment registers.
8255 */
8256 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8257 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8258 if (!(u32Eflags & X86_EFL_VM))
8259 {
8260 /* CS */
8261 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8262 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8263 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8264 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8265 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8266 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8267 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8268 /* CS cannot be loaded with NULL in protected mode. */
8269 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8270 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8271 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8272 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8273 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8274 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8275 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8276 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8277 else
8278 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8279
8280 /* SS */
8281 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8282 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8283 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8284 if ( !(pCtx->cr0 & X86_CR0_PE)
8285 || pCtx->cs.Attr.n.u4Type == 3)
8286 {
8287 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8288 }
8289 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8290 {
8291 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8292 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8293 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8294 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8295 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8296 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8297 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8298 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8299 }
8300
8301 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8302 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8303 {
8304 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8305 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8306 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8307 || pCtx->ds.Attr.n.u4Type > 11
8308 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8309 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8310 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8311 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8312 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8313 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8314 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8315 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8316 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8317 }
8318 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8319 {
8320 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8321 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8322 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8323 || pCtx->es.Attr.n.u4Type > 11
8324 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8325 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8326 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8327 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8328 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8329 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8330 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8331 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8332 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8333 }
8334 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8335 {
8336 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8337 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8338 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8339 || pCtx->fs.Attr.n.u4Type > 11
8340 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8341 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8342 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8343 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8344 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8345 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8346 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8347 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8348 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8349 }
8350 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8351 {
8352 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8353 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8354 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8355 || pCtx->gs.Attr.n.u4Type > 11
8356 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8357 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8358 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8359 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8360 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8361 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8362 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8363 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8364 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8365 }
8366 /* 64-bit capable CPUs. */
8367#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8368 if (HMVMX_IS_64BIT_HOST_MODE())
8369 {
8370 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8371 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8372 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8373 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8374 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8375 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8376 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8377 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8378 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8379 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8380 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8381 }
8382#endif
8383 }
8384 else
8385 {
8386 /* V86 mode checks. */
8387 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8388 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8389 {
8390 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8391 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8392 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8393 }
8394 else
8395 {
8396 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8397 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8398 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8399 }
8400
8401 /* CS */
8402 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8403 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8404 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8405 /* SS */
8406 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8407 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8408 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8409 /* DS */
8410 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8411 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8412 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8413 /* ES */
8414 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8415 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8416 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8417 /* FS */
8418 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8419 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8420 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8421 /* GS */
8422 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8423 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8424 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8425 /* 64-bit capable CPUs. */
8426#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8427 if (HMVMX_IS_64BIT_HOST_MODE())
8428 {
8429 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8430 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8431 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8432 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8433 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8434 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8435 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8436 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8437 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8438 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8439 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8440 }
8441#endif
8442 }
8443
8444 /*
8445 * TR.
8446 */
8447 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8448 /* 64-bit capable CPUs. */
8449#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8450 if (HMVMX_IS_64BIT_HOST_MODE())
8451 {
8452 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8453 }
8454#endif
8455 if (fLongModeGuest)
8456 {
8457 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8458 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8459 }
8460 else
8461 {
8462 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8463 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8464 VMX_IGS_TR_ATTR_TYPE_INVALID);
8465 }
8466 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8467 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8468 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8469 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8470 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8471 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8472 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8473 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8474
8475 /*
8476 * GDTR and IDTR.
8477 */
8478#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8479 if (HMVMX_IS_64BIT_HOST_MODE())
8480 {
8481 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8482 AssertRCBreak(rc);
8483 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8484
8485 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8486 AssertRCBreak(rc);
8487 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8488 }
8489#endif
8490
8491 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8492 AssertRCBreak(rc);
8493 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8494
8495 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8496 AssertRCBreak(rc);
8497 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8498
8499 /*
8500 * Guest Non-Register State.
8501 */
8502 /* Activity State. */
8503 uint32_t u32ActivityState;
8504 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8505 AssertRCBreak(rc);
8506 HMVMX_CHECK_BREAK( !u32ActivityState
8507 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8508 VMX_IGS_ACTIVITY_STATE_INVALID);
8509 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8510 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8511 uint32_t u32IntrState;
8512 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8513 AssertRCBreak(rc);
8514 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8515 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8516 {
8517 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8518 }
8519
8520 /** @todo Activity state and injecting interrupts. Left as a todo since we
8521 * currently don't use activity states but ACTIVE. */
8522
8523 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8524 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8525
8526 /* Guest interruptibility-state. */
8527 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8528 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8529 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8530 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8531 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8532 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8533 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8534 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8535 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8536 if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8537 {
8538 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8539 {
8540 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8541 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8542 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8543 }
8544 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8545 {
8546 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8547 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8548 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8549 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8550 }
8551 }
8552 /** @todo Assumes the processor is not in SMM. */
8553 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8554 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8555 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8556 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8557 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8558 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8559 && VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8560 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8561 {
8562 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8563 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8564 }
8565
8566 /* Pending debug exceptions. */
8567 if (HMVMX_IS_64BIT_HOST_MODE())
8568 {
8569 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8570 AssertRCBreak(rc);
8571 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8572 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8573 u32Val = u64Val; /* For pending debug exceptions checks below. */
8574 }
8575 else
8576 {
8577 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8578 AssertRCBreak(rc);
8579 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8580 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8581 }
8582
8583 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8584 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8585 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8586 {
8587 if ( (u32Eflags & X86_EFL_TF)
8588 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8589 {
8590 /* Bit 14 is PendingDebug.BS. */
8591 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8592 }
8593 if ( !(u32Eflags & X86_EFL_TF)
8594 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8595 {
8596 /* Bit 14 is PendingDebug.BS. */
8597 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8598 }
8599 }
8600
8601 /* VMCS link pointer. */
8602 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8603 AssertRCBreak(rc);
8604 if (u64Val != UINT64_C(0xffffffffffffffff))
8605 {
8606 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8607 /** @todo Bits beyond the processor's physical-address width MBZ. */
8608 /** @todo 32-bit located in memory referenced by value of this field (as a
8609 * physical address) must contain the processor's VMCS revision ID. */
8610 /** @todo SMM checks. */
8611 }
8612
8613 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8614
8615 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8616 if (uError == VMX_IGS_ERROR)
8617 uError = VMX_IGS_REASON_NOT_FOUND;
8618 } while (0);
8619
8620 pVCpu->hm.s.u32HMError = uError;
8621 return uError;
8622
8623#undef HMVMX_ERROR_BREAK
8624#undef HMVMX_CHECK_BREAK
8625#undef HMVMX_IS_CANONICAL
8626}
8627
8628/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8629/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8630/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8631
8632/** @name VM-exit handlers.
8633 * @{
8634 */
8635
8636/**
8637 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8638 */
8639HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8640{
8641 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8642 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8643 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8644#if HC_ARCH_BITS == 64
8645 Assert(ASMIntAreEnabled());
8646 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8647 return VINF_SUCCESS;
8648#endif
8649 return VINF_EM_RAW_INTERRUPT;
8650}
8651
8652
8653/**
8654 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8655 */
8656HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8657{
8658 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8659 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8660
8661 int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8662 AssertRCReturn(rc, rc);
8663
8664 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8665 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8666 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8667 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8668
8669 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8670 {
8671 /*
8672 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8673 * anything we inject is not going to cause a VM-exit directly for the event being injected.
8674 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8675 *
8676 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8677 */
8678 VMXDispatchHostNmi();
8679 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
8680 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8681 return VINF_SUCCESS;
8682 }
8683
8684 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8685 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8686 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8687 {
8688 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8689 return VINF_SUCCESS;
8690 }
8691 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8692 {
8693 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8694 return rc;
8695 }
8696
8697 uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8698 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8699 switch (uIntrType)
8700 {
8701 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8702 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8703 /* no break */
8704 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8705 {
8706 switch (uVector)
8707 {
8708 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8709 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8710 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8711 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8712 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8713 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8714#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8715 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8716 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8717 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8718 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8719 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8720 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8721 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8722 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8723 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8724 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8725#endif
8726 default:
8727 {
8728 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8729 AssertRCReturn(rc, rc);
8730
8731 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8732 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8733 {
8734 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8735 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8736 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8737
8738 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8739 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8740 AssertRCReturn(rc, rc);
8741 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8742 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8743 0 /* GCPtrFaultAddress */);
8744 AssertRCReturn(rc, rc);
8745 }
8746 else
8747 {
8748 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8749 pVCpu->hm.s.u32HMError = uVector;
8750 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8751 }
8752 break;
8753 }
8754 }
8755 break;
8756 }
8757
8758 default:
8759 {
8760 pVCpu->hm.s.u32HMError = uExitIntrInfo;
8761 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8762 AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8763 break;
8764 }
8765 }
8766 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8767 return rc;
8768}
8769
8770
8771/**
8772 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8773 */
8774HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8775{
8776 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8777
8778 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8779 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8780 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8781 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8782 AssertRCReturn(rc, rc);
8783
8784 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8785 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8786 return VINF_SUCCESS;
8787}
8788
8789
8790/**
8791 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8792 */
8793HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8794{
8795 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8796 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8797 pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8798 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8799}
8800
8801
8802/**
8803 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8804 */
8805HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8806{
8807 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8808 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8809 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8810}
8811
8812
8813/**
8814 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8815 */
8816HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8817{
8818 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8819 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8820 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8821}
8822
8823
8824/**
8825 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8826 */
8827HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8828{
8829 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8830 PVM pVM = pVCpu->CTX_SUFF(pVM);
8831 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8832 if (RT_LIKELY(rc == VINF_SUCCESS))
8833 {
8834 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8835 Assert(pVmxTransient->cbInstr == 2);
8836 }
8837 else
8838 {
8839 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8840 rc = VERR_EM_INTERPRETER;
8841 }
8842 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8843 return rc;
8844}
8845
8846
8847/**
8848 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8849 */
8850HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8851{
8852 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8853 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8854 AssertRCReturn(rc, rc);
8855
8856 if (pMixedCtx->cr4 & X86_CR4_SMXE)
8857 return VINF_EM_RAW_EMULATE_INSTR;
8858
8859 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8860 pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8861 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8862}
8863
8864
8865/**
8866 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8867 */
8868HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8869{
8870 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8871 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8872 AssertRCReturn(rc, rc);
8873
8874 PVM pVM = pVCpu->CTX_SUFF(pVM);
8875 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8876 if (RT_LIKELY(rc == VINF_SUCCESS))
8877 {
8878 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8879 Assert(pVmxTransient->cbInstr == 2);
8880 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8881 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8882 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8883 }
8884 else
8885 {
8886 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8887 rc = VERR_EM_INTERPRETER;
8888 }
8889 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8890 return rc;
8891}
8892
8893
8894/**
8895 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8896 */
8897HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8898{
8899 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8900 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8901 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8902 AssertRCReturn(rc, rc);
8903
8904 PVM pVM = pVCpu->CTX_SUFF(pVM);
8905 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8906 if (RT_LIKELY(rc == VINF_SUCCESS))
8907 {
8908 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8909 Assert(pVmxTransient->cbInstr == 3);
8910 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8911 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8912 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8913 }
8914 else
8915 {
8916 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8917 rc = VERR_EM_INTERPRETER;
8918 }
8919 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8920 return rc;
8921}
8922
8923
8924/**
8925 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8926 */
8927HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8928{
8929 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8930 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8931 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8932 AssertRCReturn(rc, rc);
8933
8934 PVM pVM = pVCpu->CTX_SUFF(pVM);
8935 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8936 if (RT_LIKELY(rc == VINF_SUCCESS))
8937 {
8938 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8939 Assert(pVmxTransient->cbInstr == 2);
8940 }
8941 else
8942 {
8943 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8944 rc = VERR_EM_INTERPRETER;
8945 }
8946 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8947 return rc;
8948}
8949
8950
8951/**
8952 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8953 */
8954HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8955{
8956 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8957 PVM pVM = pVCpu->CTX_SUFF(pVM);
8958 Assert(!pVM->hm.s.fNestedPaging);
8959
8960 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8961 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8962 AssertRCReturn(rc, rc);
8963
8964 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8965 rc = VBOXSTRICTRC_VAL(rc2);
8966 if (RT_LIKELY(rc == VINF_SUCCESS))
8967 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8968 else
8969 {
8970 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8971 pVmxTransient->uExitQualification, rc));
8972 }
8973 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8974 return rc;
8975}
8976
8977
8978/**
8979 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8980 */
8981HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8982{
8983 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8984 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8985 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8986 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8987 AssertRCReturn(rc, rc);
8988
8989 PVM pVM = pVCpu->CTX_SUFF(pVM);
8990 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8991 if (RT_LIKELY(rc == VINF_SUCCESS))
8992 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8993 else
8994 {
8995 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8996 rc = VERR_EM_INTERPRETER;
8997 }
8998 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8999 return rc;
9000}
9001
9002
9003/**
9004 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9005 */
9006HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9007{
9008 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9009 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9010 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9011 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9012 AssertRCReturn(rc, rc);
9013
9014 PVM pVM = pVCpu->CTX_SUFF(pVM);
9015 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9016 rc = VBOXSTRICTRC_VAL(rc2);
9017 if (RT_LIKELY( rc == VINF_SUCCESS
9018 || rc == VINF_EM_HALT))
9019 {
9020 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9021 AssertRCReturn(rc3, rc3);
9022
9023 if ( rc == VINF_EM_HALT
9024 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9025 {
9026 rc = VINF_SUCCESS;
9027 }
9028 }
9029 else
9030 {
9031 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9032 rc = VERR_EM_INTERPRETER;
9033 }
9034 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
9035 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9036 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9037 return rc;
9038}
9039
9040
9041/**
9042 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9043 */
9044HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9045{
9046 /*
9047 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9048 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9049 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
9050 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9051 */
9052 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9053 pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
9054 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9055}
9056
9057
9058/**
9059 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9060 */
9061HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9062{
9063 /*
9064 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9065 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9066 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9067 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9068 */
9069 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9070 pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
9071 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9072}
9073
9074
9075/**
9076 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9077 */
9078HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9079{
9080 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9081 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9082 pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
9083 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9084}
9085
9086
9087/**
9088 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9089 */
9090HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9091{
9092 /*
9093 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9094 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9095 * See Intel spec. 25.3 "Other Causes of VM-exits".
9096 */
9097 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9098 pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
9099 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9100}
9101
9102
9103/**
9104 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9105 * VM-exit.
9106 */
9107HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9108{
9109 /*
9110 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9111 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9112 *
9113 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9114 * See Intel spec. "23.8 Restrictions on VMX operation".
9115 */
9116 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9117 return VINF_SUCCESS;
9118}
9119
9120
9121/**
9122 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9123 * VM-exit.
9124 */
9125HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9126{
9127 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9128 return VINF_EM_RESET;
9129}
9130
9131
9132/**
9133 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9134 */
9135HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9136{
9137 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9138 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9139 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9140 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9141 AssertRCReturn(rc, rc);
9142
9143 pMixedCtx->rip++;
9144 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9145 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9146 rc = VINF_SUCCESS;
9147 else
9148 rc = VINF_EM_HALT;
9149
9150 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9151 return rc;
9152}
9153
9154
9155/**
9156 * VM-exit handler for instructions that result in a #UD exception delivered to
9157 * the guest.
9158 */
9159HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9160{
9161 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9162 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9163 return VINF_SUCCESS;
9164}
9165
9166
9167/**
9168 * VM-exit handler for expiry of the VMX preemption timer.
9169 */
9170HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9171{
9172 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9173
9174 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9175 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9176
9177 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9178 PVM pVM = pVCpu->CTX_SUFF(pVM);
9179 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9180 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9181 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9182}
9183
9184
9185/**
9186 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9187 */
9188HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9189{
9190 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9191
9192 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9193 /** @todo check if XSETBV is supported by the recompiler. */
9194 return VERR_EM_INTERPRETER;
9195}
9196
9197
9198/**
9199 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9200 */
9201HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9202{
9203 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9204
9205 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9206 /** @todo implement EMInterpretInvpcid() */
9207 return VERR_EM_INTERPRETER;
9208}
9209
9210
9211/**
9212 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9213 * Error VM-exit.
9214 */
9215HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9216{
9217 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9218 AssertRCReturn(rc, rc);
9219
9220 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9221 NOREF(uInvalidReason);
9222
9223#ifdef VBOX_STRICT
9224 uint32_t uIntrState;
9225 HMVMXHCUINTREG uHCReg;
9226 uint64_t u64Val;
9227 uint32_t u32Val;
9228
9229 rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
9230 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9231 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9232 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9233 AssertRCReturn(rc, rc);
9234
9235 Log4(("uInvalidReason %u\n", uInvalidReason));
9236 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
9237 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9238 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9239 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9240
9241 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9242 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9243 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9244 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9245 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9246 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9247 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9248 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9249 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9250 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9251 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9252 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9253#endif
9254
9255 PVM pVM = pVCpu->CTX_SUFF(pVM);
9256 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9257
9258 return VERR_VMX_INVALID_GUEST_STATE;
9259}
9260
9261
9262/**
9263 * VM-exit handler for VM-entry failure due to an MSR-load
9264 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9265 */
9266HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9267{
9268 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9269 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9270}
9271
9272
9273/**
9274 * VM-exit handler for VM-entry failure due to a machine-check event
9275 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9276 */
9277HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9278{
9279 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9280 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9281}
9282
9283
9284/**
9285 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9286 * theory.
9287 */
9288HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9289{
9290 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9291 return VERR_VMX_UNDEFINED_EXIT_CODE;
9292}
9293
9294
9295/**
9296 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9297 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9298 * Conditional VM-exit.
9299 */
9300HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9301{
9302 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9303
9304 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9305 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9306 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9307 return VERR_EM_INTERPRETER;
9308 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9309 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9310}
9311
9312
9313/**
9314 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9315 */
9316HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9317{
9318 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9319
9320 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9321 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9322 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9323 return VERR_EM_INTERPRETER;
9324 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9325 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9326}
9327
9328
9329/**
9330 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9331 */
9332HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9333{
9334 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9335
9336 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9337 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9338 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9339 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9340 AssertRCReturn(rc, rc);
9341 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9342
9343 PVM pVM = pVCpu->CTX_SUFF(pVM);
9344 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9345 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9346 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9347 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9348
9349 if (RT_LIKELY(rc == VINF_SUCCESS))
9350 {
9351 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9352 Assert(pVmxTransient->cbInstr == 2);
9353 }
9354 return rc;
9355}
9356
9357
9358/**
9359 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9360 */
9361HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9362{
9363 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9364 PVM pVM = pVCpu->CTX_SUFF(pVM);
9365 int rc = VINF_SUCCESS;
9366
9367 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9368 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9369 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9370 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9371 AssertRCReturn(rc, rc);
9372 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9373
9374 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9375 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9376 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9377
9378 if (RT_LIKELY(rc == VINF_SUCCESS))
9379 {
9380 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9381
9382 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9383 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9384 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9385 {
9386 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9387 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9388 EMInterpretWrmsr() changes it. */
9389 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9390 }
9391 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9392 {
9393 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9394 AssertRCReturn(rc, rc);
9395 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
9396 }
9397 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9398 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9399
9400 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9401 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9402 {
9403 switch (pMixedCtx->ecx)
9404 {
9405 case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
9406 case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
9407 case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
9408 case MSR_K8_FS_BASE: /* no break */
9409 case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS; break;
9410 case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
9411 }
9412 }
9413#ifdef VBOX_STRICT
9414 else
9415 {
9416 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9417 switch (pMixedCtx->ecx)
9418 {
9419 case MSR_IA32_SYSENTER_CS:
9420 case MSR_IA32_SYSENTER_EIP:
9421 case MSR_IA32_SYSENTER_ESP:
9422 case MSR_K8_FS_BASE:
9423 case MSR_K8_GS_BASE:
9424 {
9425 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9426 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9427 }
9428
9429 case MSR_K8_LSTAR:
9430 case MSR_K6_STAR:
9431 case MSR_K8_SF_MASK:
9432 case MSR_K8_TSC_AUX:
9433 case MSR_K8_KERNEL_GS_BASE:
9434 {
9435 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9436 pMixedCtx->ecx));
9437 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9438 }
9439 }
9440 }
9441#endif /* VBOX_STRICT */
9442 }
9443 return rc;
9444}
9445
9446
9447/**
9448 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9449 */
9450HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9451{
9452 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9453
9454 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9455 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9456 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9457 return VERR_EM_INTERPRETER;
9458 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9459 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9460}
9461
9462
9463/**
9464 * VM-exit handler for when the TPR value is lowered below the specified
9465 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9466 */
9467HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9468{
9469 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9470 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9471
9472 /*
9473 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9474 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9475 * resume guest execution.
9476 */
9477 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9478 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9479 return VINF_SUCCESS;
9480}
9481
9482
9483/**
9484 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9485 * VM-exit.
9486 *
9487 * @retval VINF_SUCCESS when guest execution can continue.
9488 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9489 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9490 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9491 * recompiler.
9492 */
9493HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9494{
9495 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9496 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9497 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9498 AssertRCReturn(rc, rc);
9499
9500 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9501 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9502 PVM pVM = pVCpu->CTX_SUFF(pVM);
9503 switch (uAccessType)
9504 {
9505 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9506 {
9507#if 0
9508 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9509 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9510#else
9511 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9512 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9513 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9514#endif
9515 AssertRCReturn(rc, rc);
9516
9517 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9518 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9519 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9520 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9521
9522 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9523 {
9524 case 0: /* CR0 */
9525 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9526 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9527 break;
9528 case 2: /* C2 **/
9529 /* Nothing to do here, CR2 it's not part of the VMCS. */
9530 break;
9531 case 3: /* CR3 */
9532 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9533 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9534 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR3;
9535 break;
9536 case 4: /* CR4 */
9537 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9538 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR4;
9539 break;
9540 case 8: /* CR8 */
9541 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9542 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9543 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9544 break;
9545 default:
9546 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9547 break;
9548 }
9549
9550 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9551 break;
9552 }
9553
9554 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9555 {
9556 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9557 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9558 AssertRCReturn(rc, rc);
9559 Assert( !pVM->hm.s.fNestedPaging
9560 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9561 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9562
9563 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9564 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9565 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9566
9567 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9568 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9569 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9570 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9571 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9572 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9573 break;
9574 }
9575
9576 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9577 {
9578 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9579 AssertRCReturn(rc, rc);
9580 rc = EMInterpretCLTS(pVM, pVCpu);
9581 AssertRCReturn(rc, rc);
9582 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9583 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9584 Log4(("CRX CLTS write rc=%d\n", rc));
9585 break;
9586 }
9587
9588 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9589 {
9590 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9591 AssertRCReturn(rc, rc);
9592 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9593 if (RT_LIKELY(rc == VINF_SUCCESS))
9594 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9595 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9596 Log4(("CRX LMSW write rc=%d\n", rc));
9597 break;
9598 }
9599
9600 default:
9601 {
9602 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9603 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9604 }
9605 }
9606
9607 /* Validate possible error codes. */
9608 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9609 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9610 if (RT_SUCCESS(rc))
9611 {
9612 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9613 AssertRCReturn(rc2, rc2);
9614 }
9615
9616 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9617 return rc;
9618}
9619
9620
9621/**
9622 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9623 * VM-exit.
9624 */
9625HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9626{
9627 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9628 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9629
9630 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9631 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9632 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9633 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9634 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9635 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9636 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9637 AssertRCReturn(rc2, rc2);
9638
9639 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9640 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9641 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9642 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9643 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9644 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9645 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9646
9647 /* I/O operation lookup arrays. */
9648 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9649 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9650
9651 VBOXSTRICTRC rcStrict;
9652 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9653 const uint32_t cbInstr = pVmxTransient->cbInstr;
9654 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9655 PVM pVM = pVCpu->CTX_SUFF(pVM);
9656 if (fIOString)
9657 {
9658 /*
9659 * INS/OUTS - I/O String instruction.
9660 *
9661 * Use instruction-information if available, otherwise fall back on
9662 * interpreting the instruction.
9663 */
9664 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9665#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9666 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9667 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9668 {
9669 rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9670 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9671 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9672 AssertRCReturn(rc2, rc2);
9673 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9674 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9675 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9676 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9677 if (fIOWrite)
9678 {
9679 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9680 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9681 //if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9682 // hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9683 // pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9684 }
9685 else
9686 {
9687 AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9688 ("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9689 VERR_HMVMX_IPE_4);
9690 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9691 //if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9692 // hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9693 }
9694 }
9695 else
9696 {
9697 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9698 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9699 AssertRCReturn(rc2, rc2);
9700 rcStrict = IEMExecOne(pVCpu);
9701 }
9702 /** @todo IEM needs to be setting these flags somehow. */
9703 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9704 fUpdateRipAlready = true;
9705#else
9706 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9707 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9708 if (RT_SUCCESS(rcStrict))
9709 {
9710 if (fIOWrite)
9711 {
9712 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9713 (DISCPUMODE)pDis->uAddrMode, cbValue);
9714 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9715 }
9716 else
9717 {
9718 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9719 (DISCPUMODE)pDis->uAddrMode, cbValue);
9720 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9721 }
9722 }
9723 else
9724 {
9725 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9726 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9727 }
9728#endif
9729 }
9730 else
9731 {
9732 /*
9733 * IN/OUT - I/O instruction.
9734 */
9735 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9736 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9737 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9738 if (fIOWrite)
9739 {
9740 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9741 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9742 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9743 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9744 }
9745 else
9746 {
9747 uint32_t u32Result = 0;
9748 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9749 if (IOM_SUCCESS(rcStrict))
9750 {
9751 /* Save result of I/O IN instr. in AL/AX/EAX. */
9752 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
9753 }
9754 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9755 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9756 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9757 }
9758 }
9759
9760 if (IOM_SUCCESS(rcStrict))
9761 {
9762 if (!fUpdateRipAlready)
9763 {
9764 pMixedCtx->rip += cbInstr;
9765 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9766 }
9767
9768 /*
9769 * If any I/O breakpoints are armed, we need to check if one triggered
9770 * and take appropriate action.
9771 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9772 */
9773 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9774 AssertRCReturn(rc2, rc2);
9775
9776 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9777 * execution engines about whether hyper BPs and such are pending. */
9778 uint32_t const uDr7 = pMixedCtx->dr[7];
9779 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9780 && X86_DR7_ANY_RW_IO(uDr7)
9781 && (pMixedCtx->cr4 & X86_CR4_DE))
9782 || DBGFBpIsHwIoArmed(pVM)))
9783 {
9784 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9785
9786 /* We're playing with the host CPU state here, make sure we don't preempt. */
9787 HM_DISABLE_PREEMPT_IF_NEEDED();
9788 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
9789
9790 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9791 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9792 {
9793 /* Raise #DB. */
9794 if (fIsGuestDbgActive)
9795 ASMSetDR6(pMixedCtx->dr[6]);
9796 if (pMixedCtx->dr[7] != uDr7)
9797 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9798
9799 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9800 }
9801 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9802 else if ( rcStrict2 != VINF_SUCCESS
9803 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
9804 rcStrict = rcStrict2;
9805
9806 HM_RESTORE_PREEMPT_IF_NEEDED();
9807 }
9808 }
9809
9810#ifdef DEBUG
9811 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9812 Assert(!fIOWrite);
9813 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9814 Assert(fIOWrite);
9815 else
9816 {
9817 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9818 * statuses, that the VMM device and some others may return. See
9819 * IOM_SUCCESS() for guidance. */
9820 AssertMsg( RT_FAILURE(rcStrict)
9821 || rcStrict == VINF_SUCCESS
9822 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
9823 || rcStrict == VINF_EM_DBG_BREAKPOINT
9824 || rcStrict == VINF_EM_RAW_GUEST_TRAP
9825 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9826 }
9827#endif
9828
9829 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9830 return VBOXSTRICTRC_TODO(rcStrict);
9831}
9832
9833
9834/**
9835 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9836 * VM-exit.
9837 */
9838HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9839{
9840 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9841
9842 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
9843 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9844 AssertRCReturn(rc, rc);
9845 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9846 {
9847 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9848 AssertRCReturn(rc, rc);
9849 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9850 {
9851 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9852
9853 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9854 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9855 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9856 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9857 {
9858 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9859 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9860
9861 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9862 Assert(!pVCpu->hm.s.Event.fPending);
9863 pVCpu->hm.s.Event.fPending = true;
9864 pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9865 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9866 AssertRCReturn(rc, rc);
9867 if (fErrorCodeValid)
9868 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9869 else
9870 pVCpu->hm.s.Event.u32ErrCode = 0;
9871 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9872 && uVector == X86_XCPT_PF)
9873 {
9874 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9875 }
9876
9877 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9878 }
9879 }
9880 }
9881
9882 /** @todo Emulate task switch someday, currently just going back to ring-3 for
9883 * emulation. */
9884 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9885 return VERR_EM_INTERPRETER;
9886}
9887
9888
9889/**
9890 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9891 */
9892HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9893{
9894 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9895 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9896 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9897 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9898 AssertRCReturn(rc, rc);
9899 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9900 return VINF_EM_DBG_STEPPED;
9901}
9902
9903
9904/**
9905 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9906 */
9907HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9908{
9909 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9910
9911 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9912 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9913 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9914 return VINF_SUCCESS;
9915 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9916 return rc;
9917
9918#if 0
9919 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9920 * just sync the whole thing. */
9921 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9922#else
9923 /* Aggressive state sync. for now. */
9924 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9925 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9926 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9927#endif
9928 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9929 AssertRCReturn(rc, rc);
9930
9931 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9932 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9933 switch (uAccessType)
9934 {
9935 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9936 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9937 {
9938 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9939 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9940 {
9941 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9942 }
9943
9944 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9945 GCPhys &= PAGE_BASE_GC_MASK;
9946 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9947 PVM pVM = pVCpu->CTX_SUFF(pVM);
9948 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9949 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9950
9951 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9952 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9953 CPUMCTX2CORE(pMixedCtx), GCPhys);
9954 rc = VBOXSTRICTRC_VAL(rc2);
9955 Log4(("ApicAccess rc=%d\n", rc));
9956 if ( rc == VINF_SUCCESS
9957 || rc == VERR_PAGE_TABLE_NOT_PRESENT
9958 || rc == VERR_PAGE_NOT_PRESENT)
9959 {
9960 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
9961 | HM_CHANGED_VMX_GUEST_APIC_STATE;
9962 rc = VINF_SUCCESS;
9963 }
9964 break;
9965 }
9966
9967 default:
9968 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9969 rc = VINF_EM_RAW_EMULATE_INSTR;
9970 break;
9971 }
9972
9973 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9974 return rc;
9975}
9976
9977
9978/**
9979 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9980 * VM-exit.
9981 */
9982HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9983{
9984 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9985
9986 /* We should -not- get this VM-exit if the guest's debug registers are active. See CPUMR0LoadGuestDebugState(). */
9987#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9988 if ( !CPUMIsGuestInLongModeEx(pMixedCtx) /* EFER is always up-to-date. */
9989 && CPUMIsGuestDebugStateActive(pVCpu))
9990#else
9991 if (CPUMIsGuestDebugStateActive(pVCpu))
9992#endif
9993 {
9994 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9995 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9996 }
9997
9998 int rc = VERR_INTERNAL_ERROR_5;
9999 if ( !DBGFIsStepping(pVCpu)
10000 && !pVCpu->hm.s.fSingleInstruction
10001 && !CPUMIsHyperDebugStateActive(pVCpu))
10002 {
10003 /* Don't intercept MOV DRx and #DB any more. */
10004 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
10005 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10006 AssertRCReturn(rc, rc);
10007
10008 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10009 {
10010#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10011 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
10012 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
10013 AssertRCReturn(rc, rc);
10014#endif
10015 }
10016
10017 /* We're playing with the host CPU state here, make sure we can't preempt. */
10018 HM_DISABLE_PREEMPT_IF_NEEDED();
10019
10020 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
10021 PVM pVM = pVCpu->CTX_SUFF(pVM);
10022 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
10023 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
10024
10025 HM_RESTORE_PREEMPT_IF_NEEDED();
10026
10027#ifdef VBOX_WITH_STATISTICS
10028 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10029 AssertRCReturn(rc, rc);
10030 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10031 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10032 else
10033 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10034#endif
10035 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
10036 return VINF_SUCCESS;
10037 }
10038
10039 /*
10040 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
10041 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
10042 */
10043 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10044 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10045 AssertRCReturn(rc, rc);
10046 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10047
10048 PVM pVM = pVCpu->CTX_SUFF(pVM);
10049 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
10050 {
10051 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10052 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10053 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10054 if (RT_SUCCESS(rc))
10055 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
10056 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10057 }
10058 else
10059 {
10060 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10061 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10062 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10063 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10064 }
10065
10066 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10067 if (RT_SUCCESS(rc))
10068 {
10069 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10070 AssertRCReturn(rc2, rc2);
10071 }
10072 return rc;
10073}
10074
10075
10076/**
10077 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10078 * Conditional VM-exit.
10079 */
10080HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10081{
10082 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10083 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10084
10085 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10086 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10087 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10088 return VINF_SUCCESS;
10089 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10090 return rc;
10091
10092 RTGCPHYS GCPhys = 0;
10093 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10094
10095#if 0
10096 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10097#else
10098 /* Aggressive state sync. for now. */
10099 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10100 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10101 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10102#endif
10103 AssertRCReturn(rc, rc);
10104
10105 /*
10106 * If we succeed, resume guest execution.
10107 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10108 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10109 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10110 * weird case. See @bugref{6043}.
10111 */
10112 PVM pVM = pVCpu->CTX_SUFF(pVM);
10113 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10114 rc = VBOXSTRICTRC_VAL(rc2);
10115 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10116 if ( rc == VINF_SUCCESS
10117 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10118 || rc == VERR_PAGE_NOT_PRESENT)
10119 {
10120 /* Successfully handled MMIO operation. */
10121 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10122 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10123 rc = VINF_SUCCESS;
10124 }
10125 return rc;
10126}
10127
10128
10129/**
10130 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10131 * VM-exit.
10132 */
10133HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10134{
10135 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10136 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10137
10138 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10139 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10140 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10141 return VINF_SUCCESS;
10142 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10143 return rc;
10144
10145 RTGCPHYS GCPhys = 0;
10146 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10147 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10148#if 0
10149 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10150#else
10151 /* Aggressive state sync. for now. */
10152 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10153 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10154 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10155#endif
10156 AssertRCReturn(rc, rc);
10157
10158 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10159 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10160
10161 RTGCUINT uErrorCode = 0;
10162 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10163 uErrorCode |= X86_TRAP_PF_ID;
10164 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10165 uErrorCode |= X86_TRAP_PF_RW;
10166 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10167 uErrorCode |= X86_TRAP_PF_P;
10168
10169 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10170
10171 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10172 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10173
10174 /* Handle the pagefault trap for the nested shadow table. */
10175 PVM pVM = pVCpu->CTX_SUFF(pVM);
10176 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10177 TRPMResetTrap(pVCpu);
10178
10179 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10180 if ( rc == VINF_SUCCESS
10181 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10182 || rc == VERR_PAGE_NOT_PRESENT)
10183 {
10184 /* Successfully synced our nested page tables. */
10185 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10186 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10187 return VINF_SUCCESS;
10188 }
10189
10190 Log4(("EPT return to ring-3 rc=%d\n"));
10191 return rc;
10192}
10193
10194/** @} */
10195
10196/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10197/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10198/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10199
10200/** @name VM-exit exception handlers.
10201 * @{
10202 */
10203
10204/**
10205 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10206 */
10207static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10208{
10209 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10210 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10211
10212 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10213 AssertRCReturn(rc, rc);
10214
10215 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10216 {
10217 /* Old-style FPU error reporting needs some extra work. */
10218 /** @todo don't fall back to the recompiler, but do it manually. */
10219 return VERR_EM_INTERPRETER;
10220 }
10221
10222 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10223 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10224 return rc;
10225}
10226
10227
10228/**
10229 * VM-exit exception handler for #BP (Breakpoint exception).
10230 */
10231static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10232{
10233 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10235
10236 /** @todo Try optimize this by not saving the entire guest state unless
10237 * really needed. */
10238 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10239 AssertRCReturn(rc, rc);
10240
10241 PVM pVM = pVCpu->CTX_SUFF(pVM);
10242 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10243 if (rc == VINF_EM_RAW_GUEST_TRAP)
10244 {
10245 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10246 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10247 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10248 AssertRCReturn(rc, rc);
10249
10250 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10251 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10252 }
10253
10254 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10255 return rc;
10256}
10257
10258
10259/**
10260 * VM-exit exception handler for #DB (Debug exception).
10261 */
10262static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10263{
10264 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10265 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10266 Log6(("XcptDB\n"));
10267
10268 /*
10269 * Get the DR6-like values from the exit qualification and pass it to DBGF
10270 * for processing.
10271 */
10272 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10273 AssertRCReturn(rc, rc);
10274
10275 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10276 uint64_t uDR6 = X86_DR6_INIT_VAL;
10277 uDR6 |= ( pVmxTransient->uExitQualification
10278 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10279
10280 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10281 if (rc == VINF_EM_RAW_GUEST_TRAP)
10282 {
10283 /*
10284 * The exception was for the guest. Update DR6, DR7.GD and
10285 * IA32_DEBUGCTL.LBR before forwarding it.
10286 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10287 */
10288 HM_DISABLE_PREEMPT_IF_NEEDED();
10289
10290 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10291 pMixedCtx->dr[6] |= uDR6;
10292 if (CPUMIsGuestDebugStateActive(pVCpu))
10293 ASMSetDR6(pMixedCtx->dr[6]);
10294
10295 HM_RESTORE_PREEMPT_IF_NEEDED();
10296
10297 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10298 AssertRCReturn(rc, rc);
10299
10300 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10301 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10302
10303 /* Paranoia. */
10304 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10305 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10306
10307 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10308 AssertRCReturn(rc, rc);
10309
10310 /*
10311 * Raise #DB in the guest.
10312 */
10313 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10314 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10315 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10316 AssertRCReturn(rc, rc);
10317 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10318 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10319 return VINF_SUCCESS;
10320 }
10321
10322 /*
10323 * Not a guest trap, must be a hypervisor related debug event then.
10324 * Update DR6 in case someone is interested in it.
10325 */
10326 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10327 AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
10328 CPUMSetHyperDR6(pVCpu, uDR6);
10329
10330 return rc;
10331}
10332
10333
10334/**
10335 * VM-exit exception handler for #NM (Device-not-available exception: floating
10336 * point exception).
10337 */
10338static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10339{
10340 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10341
10342 /* We require CR0 and EFER. EFER is always up-to-date. */
10343 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10344 AssertRCReturn(rc, rc);
10345
10346 /* We're playing with the host CPU state here, have to disable preemption. */
10347 HM_DISABLE_PREEMPT_IF_NEEDED();
10348
10349#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10350 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10351 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
10352#endif
10353
10354 /* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10355 PVM pVM = pVCpu->CTX_SUFF(pVM);
10356 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
10357 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
10358
10359 if (rc == VINF_SUCCESS)
10360 {
10361 Assert(CPUMIsGuestFPUStateActive(pVCpu));
10362 HM_RESTORE_PREEMPT_IF_NEEDED();
10363
10364 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10365 return VINF_SUCCESS;
10366 }
10367 HM_RESTORE_PREEMPT_IF_NEEDED();
10368
10369 /* Forward #NM to the guest. */
10370 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10371 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10372 AssertRCReturn(rc, rc);
10373 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10374 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10375 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10376 return rc;
10377}
10378
10379
10380/**
10381 * VM-exit exception handler for #GP (General-protection exception).
10382 *
10383 * @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
10384 */
10385static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10386{
10387 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10388 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10389
10390 int rc = VERR_INTERNAL_ERROR_5;
10391 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10392 {
10393#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10394 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10395 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10396 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10397 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10398 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10399 AssertRCReturn(rc, rc);
10400 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10401 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10402 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10403 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10404 return rc;
10405#else
10406 /* We don't intercept #GP. */
10407 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10408 return VERR_VMX_UNEXPECTED_EXCEPTION;
10409#endif
10410 }
10411
10412 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10413 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10414
10415 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10416 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10417 AssertRCReturn(rc, rc);
10418
10419 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10420 uint32_t cbOp = 0;
10421 PVM pVM = pVCpu->CTX_SUFF(pVM);
10422 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10423 if (RT_SUCCESS(rc))
10424 {
10425 rc = VINF_SUCCESS;
10426 Assert(cbOp == pDis->cbInstr);
10427 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10428 switch (pDis->pCurInstr->uOpcode)
10429 {
10430 case OP_CLI:
10431 {
10432 pMixedCtx->eflags.Bits.u1IF = 0;
10433 pMixedCtx->rip += pDis->cbInstr;
10434 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10435 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10436 break;
10437 }
10438
10439 case OP_STI:
10440 {
10441 pMixedCtx->eflags.Bits.u1IF = 1;
10442 pMixedCtx->rip += pDis->cbInstr;
10443 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10444 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10445 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10446 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10447 break;
10448 }
10449
10450 case OP_HLT:
10451 {
10452 rc = VINF_EM_HALT;
10453 pMixedCtx->rip += pDis->cbInstr;
10454 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
10455 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10456 break;
10457 }
10458
10459 case OP_POPF:
10460 {
10461 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10462 uint32_t cbParm = 0;
10463 uint32_t uMask = 0;
10464 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10465 {
10466 cbParm = 4;
10467 uMask = 0xffffffff;
10468 }
10469 else
10470 {
10471 cbParm = 2;
10472 uMask = 0xffff;
10473 }
10474
10475 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10476 RTGCPTR GCPtrStack = 0;
10477 X86EFLAGS Eflags;
10478 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10479 &GCPtrStack);
10480 if (RT_SUCCESS(rc))
10481 {
10482 Assert(sizeof(Eflags.u32) >= cbParm);
10483 Eflags.u32 = 0;
10484 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10485 }
10486 if (RT_FAILURE(rc))
10487 {
10488 rc = VERR_EM_INTERPRETER;
10489 break;
10490 }
10491 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10492 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10493 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10494 /* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10495 pMixedCtx->eflags.Bits.u1RF = 0;
10496 pMixedCtx->esp += cbParm;
10497 pMixedCtx->esp &= uMask;
10498 pMixedCtx->rip += pDis->cbInstr;
10499 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10500 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10501 break;
10502 }
10503
10504 case OP_PUSHF:
10505 {
10506 uint32_t cbParm = 0;
10507 uint32_t uMask = 0;
10508 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10509 {
10510 cbParm = 4;
10511 uMask = 0xffffffff;
10512 }
10513 else
10514 {
10515 cbParm = 2;
10516 uMask = 0xffff;
10517 }
10518
10519 /* Get the stack pointer & push the contents of eflags onto the stack. */
10520 RTGCPTR GCPtrStack = 0;
10521 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10522 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10523 if (RT_FAILURE(rc))
10524 {
10525 rc = VERR_EM_INTERPRETER;
10526 break;
10527 }
10528 X86EFLAGS Eflags = pMixedCtx->eflags;
10529 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10530 Eflags.Bits.u1RF = 0;
10531 Eflags.Bits.u1VM = 0;
10532
10533 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10534 if (RT_FAILURE(rc))
10535 {
10536 rc = VERR_EM_INTERPRETER;
10537 break;
10538 }
10539 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10540 pMixedCtx->esp -= cbParm;
10541 pMixedCtx->esp &= uMask;
10542 pMixedCtx->rip += pDis->cbInstr;
10543 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP;
10544 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10545 break;
10546 }
10547
10548 case OP_IRET:
10549 {
10550 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10551 * instruction reference. */
10552 RTGCPTR GCPtrStack = 0;
10553 uint32_t uMask = 0xffff;
10554 uint16_t aIretFrame[3];
10555 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10556 {
10557 rc = VERR_EM_INTERPRETER;
10558 break;
10559 }
10560 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10561 &GCPtrStack);
10562 if (RT_SUCCESS(rc))
10563 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10564 if (RT_FAILURE(rc))
10565 {
10566 rc = VERR_EM_INTERPRETER;
10567 break;
10568 }
10569 pMixedCtx->eip = 0;
10570 pMixedCtx->ip = aIretFrame[0];
10571 pMixedCtx->cs.Sel = aIretFrame[1];
10572 pMixedCtx->cs.ValidSel = aIretFrame[1];
10573 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10574 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10575 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10576 pMixedCtx->sp += sizeof(aIretFrame);
10577 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_SEGMENT_REGS | HM_CHANGED_GUEST_RSP
10578 | HM_CHANGED_GUEST_RFLAGS;
10579 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10580 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10581 break;
10582 }
10583
10584 case OP_INT:
10585 {
10586 uint16_t uVector = pDis->Param1.uValue & 0xff;
10587 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10588 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10589 break;
10590 }
10591
10592 case OP_INTO:
10593 {
10594 if (pMixedCtx->eflags.Bits.u1OF)
10595 {
10596 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10597 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10598 }
10599 break;
10600 }
10601
10602 default:
10603 {
10604 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10605 EMCODETYPE_SUPERVISOR);
10606 rc = VBOXSTRICTRC_VAL(rc2);
10607 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
10608 Log4(("#GP rc=%Rrc\n", rc));
10609 break;
10610 }
10611 }
10612 }
10613 else
10614 rc = VERR_EM_INTERPRETER;
10615
10616 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10617 ("#GP Unexpected rc=%Rrc\n", rc));
10618 return rc;
10619}
10620
10621
10622/**
10623 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10624 * the exception reported in the VMX transient structure back into the VM.
10625 *
10626 * @remarks Requires uExitIntrInfo in the VMX transient structure to be
10627 * up-to-date.
10628 */
10629static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10630{
10631 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10632
10633 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10634 hmR0VmxCheckExitDueToEventDelivery(). */
10635 int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10636 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10637 AssertRCReturn(rc, rc);
10638 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10639
10640 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10641 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10642 return VINF_SUCCESS;
10643}
10644
10645
10646/**
10647 * VM-exit exception handler for #PF (Page-fault exception).
10648 */
10649static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10650{
10651 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10652 PVM pVM = pVCpu->CTX_SUFF(pVM);
10653 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10654 rc |= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10655 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10656 AssertRCReturn(rc, rc);
10657
10658#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
10659 if (pVM->hm.s.fNestedPaging)
10660 {
10661 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10662 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10663 {
10664 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10665 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10666 0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10667 }
10668 else
10669 {
10670 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10671 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10672 Log4(("Pending #DF due to vectoring #PF. NP\n"));
10673 }
10674 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10675 return rc;
10676 }
10677#else
10678 Assert(!pVM->hm.s.fNestedPaging);
10679#endif
10680
10681 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10682 AssertRCReturn(rc, rc);
10683
10684 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10685 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10686
10687 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10688 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10689 (RTGCPTR)pVmxTransient->uExitQualification);
10690
10691 Log4(("#PF: rc=%Rrc\n", rc));
10692 if (rc == VINF_SUCCESS)
10693 {
10694 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10695 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
10696 * memory? We don't update the whole state here... */
10697 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10698 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10699 TRPMResetTrap(pVCpu);
10700 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10701 return rc;
10702 }
10703 else if (rc == VINF_EM_RAW_GUEST_TRAP)
10704 {
10705 if (!pVmxTransient->fVectoringPF)
10706 {
10707 /* It's a guest page fault and needs to be reflected to the guest. */
10708 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10709 TRPMResetTrap(pVCpu);
10710 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10711 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10712 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10713 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10714 }
10715 else
10716 {
10717 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10718 TRPMResetTrap(pVCpu);
10719 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10720 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10721 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10722 }
10723
10724 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10725 return VINF_SUCCESS;
10726 }
10727
10728 TRPMResetTrap(pVCpu);
10729 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10730 return rc;
10731}
10732
10733/** @} */
10734
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette