VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 48448

Last change on this file since 48448 was 48443, checked in by vboxsync, 12 years ago

VMM/HMVMXR0: Leave VT-x properly for VERR_VMX_INVALID_GUEST_STATE and VERR_VMX_INVALID_VMCS_PTR gurus.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 447.9 KB
Line 
1/* $Id: HMVMXR0.cpp 48443 2013-09-12 09:46:35Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#endif
45
46
47/*******************************************************************************
48* Defined Constants And Macros *
49*******************************************************************************/
50#if defined(RT_ARCH_AMD64)
51# define HMVMX_IS_64BIT_HOST_MODE() (true)
52typedef RTHCUINTREG HMVMXHCUINTREG;
53#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54extern "C" uint32_t g_fVMXIs64bitHost;
55# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56typedef uint64_t HMVMXHCUINTREG;
57#else
58# define HMVMX_IS_64BIT_HOST_MODE() (false)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#endif
61
62/** Use the function table. */
63#define HMVMX_USE_FUNCTION_TABLE
64
65/** Determine which tagged-TLB flush handler to use. */
66#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71/** @name Updated-guest-state flags.
72 * @{ */
73#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94 | HMVMX_UPDATED_GUEST_RSP \
95 | HMVMX_UPDATED_GUEST_RFLAGS \
96 | HMVMX_UPDATED_GUEST_CR0 \
97 | HMVMX_UPDATED_GUEST_CR3 \
98 | HMVMX_UPDATED_GUEST_CR4 \
99 | HMVMX_UPDATED_GUEST_GDTR \
100 | HMVMX_UPDATED_GUEST_IDTR \
101 | HMVMX_UPDATED_GUEST_LDTR \
102 | HMVMX_UPDATED_GUEST_TR \
103 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104 | HMVMX_UPDATED_GUEST_DEBUG \
105 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112 | HMVMX_UPDATED_GUEST_APIC_STATE)
113/** @} */
114
115/** @name
116 * Flags to skip redundant reads of some common VMCS fields that are not part of
117 * the guest-CPU state but are in the transient structure.
118 */
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually (except #PF). #NM is also
143 * handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
144 * intercepted even in real-mode if we have Nested Paging support.
145 */
146#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
147 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
148 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
149 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
150 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
151 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
152 | RT_BIT(X86_XCPT_XF))
153
154/**
155 * Exception bitmap mask for all contributory exceptions.
156 *
157 * Page fault is deliberately excluded here as it's conditional as to whether
158 * it's contributory or benign. Page faults are handled separately.
159 */
160#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
161 | RT_BIT(X86_XCPT_DE))
162
163/** Maximum VM-instruction error number. */
164#define HMVMX_INSTR_ERROR_MAX 28
165
166/** Profiling macro. */
167#ifdef HM_PROFILE_EXIT_DISPATCH
168# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
169# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
170#else
171# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
173#endif
174
175/** Assert that preemption is disabled or covered by thread-context hooks. */
176#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
177 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
178
179/** Assert that we haven't migrated CPUs when thread-context hooks are not
180 * used. */
181#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
182 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
183 ("Illegal migration! Entered on CPU %u Current %u\n", \
184 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
185
186/*******************************************************************************
187* Structures and Typedefs *
188*******************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG uEflags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
203 uint64_t u64LStarMsr;
204 /** The guest's TPR value used for TPR shadowing. */
205 uint8_t u8GuestTpr;
206 /** Alignment. */
207 uint8_t abAlignment0[7];
208
209 /** The basic VM-exit reason. */
210 uint16_t uExitReason;
211 /** Alignment. */
212 uint16_t u16Alignment0;
213 /** The VM-exit interruption error code. */
214 uint32_t uExitIntrErrorCode;
215 /** The VM-exit exit qualification. */
216 uint64_t uExitQualification;
217
218 /** The VM-exit interruption-information field. */
219 uint32_t uExitIntrInfo;
220 /** The VM-exit instruction-length field. */
221 uint32_t cbInstr;
222 /** The VM-exit instruction-information field. */
223 union
224 {
225 /** Plain unsigned int representation. */
226 uint32_t u;
227 /** INS and OUTS information. */
228 struct
229 {
230 uint32_t u6Reserved0 : 6;
231 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
232 uint32_t u3AddrSize : 3;
233 uint32_t u5Reserved1 : 5;
234 /** The segment register (X86_SREG_XXX). */
235 uint32_t iSegReg : 3;
236 uint32_t uReserved2 : 14;
237 } StrIo;
238 } ExitInstrInfo;
239 /** Whether the VM-entry failed or not. */
240 bool fVMEntryFailed;
241 /** Alignment. */
242 uint8_t abAlignment1[3];
243
244 /** The VM-entry interruption-information field. */
245 uint32_t uEntryIntrInfo;
246 /** The VM-entry exception error code field. */
247 uint32_t uEntryXcptErrorCode;
248 /** The VM-entry instruction length field. */
249 uint32_t cbEntryInstr;
250
251 /** IDT-vectoring information field. */
252 uint32_t uIdtVectoringInfo;
253 /** IDT-vectoring error code. */
254 uint32_t uIdtVectoringErrorCode;
255
256 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
257 uint32_t fVmcsFieldsRead;
258 /** Whether TSC-offsetting should be setup before VM-entry. */
259 bool fUpdateTscOffsettingAndPreemptTimer;
260 /** Whether the VM-exit was caused by a page-fault during delivery of a
261 * contributory exception or a page-fault. */
262 bool fVectoringPF;
263} VMXTRANSIENT;
264AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
265AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
266AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
267AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
268/** Pointer to VMX transient state. */
269typedef VMXTRANSIENT *PVMXTRANSIENT;
270
271
272/**
273 * MSR-bitmap read permissions.
274 */
275typedef enum VMXMSREXITREAD
276{
277 /** Reading this MSR causes a VM-exit. */
278 VMXMSREXIT_INTERCEPT_READ = 0xb,
279 /** Reading this MSR does not cause a VM-exit. */
280 VMXMSREXIT_PASSTHRU_READ
281} VMXMSREXITREAD;
282
283/**
284 * MSR-bitmap write permissions.
285 */
286typedef enum VMXMSREXITWRITE
287{
288 /** Writing to this MSR causes a VM-exit. */
289 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
290 /** Writing to this MSR does not cause a VM-exit. */
291 VMXMSREXIT_PASSTHRU_WRITE
292} VMXMSREXITWRITE;
293
294/**
295 * VMX VM-exit handler.
296 *
297 * @returns VBox status code.
298 * @param pVCpu Pointer to the VMCPU.
299 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
300 * out-of-sync. Make sure to update the required
301 * fields before using them.
302 * @param pVmxTransient Pointer to the VMX-transient structure.
303 */
304#ifndef HMVMX_USE_FUNCTION_TABLE
305typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306#else
307typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308/** Pointer to VM-exit handler. */
309typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
310#endif
311
312
313/*******************************************************************************
314* Internal Functions *
315*******************************************************************************/
316static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
317static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
318static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
319static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
320 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
321#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
322static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
323#endif
324#ifndef HMVMX_USE_FUNCTION_TABLE
325DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
326# define HMVMX_EXIT_DECL static int
327#else
328# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
329#endif
330
331/** @name VM-exit handlers.
332 * @{
333 */
334static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
335static FNVMXEXITHANDLER hmR0VmxExitExtInt;
336static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
337static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
338static FNVMXEXITHANDLER hmR0VmxExitSipi;
339static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
340static FNVMXEXITHANDLER hmR0VmxExitSmi;
341static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
342static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
343static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
344static FNVMXEXITHANDLER hmR0VmxExitCpuid;
345static FNVMXEXITHANDLER hmR0VmxExitGetsec;
346static FNVMXEXITHANDLER hmR0VmxExitHlt;
347static FNVMXEXITHANDLER hmR0VmxExitInvd;
348static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
349static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
350static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
351static FNVMXEXITHANDLER hmR0VmxExitRsm;
352static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
353static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
354static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
355static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
356static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
357static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
358static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
359static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
360static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
361static FNVMXEXITHANDLER hmR0VmxExitMwait;
362static FNVMXEXITHANDLER hmR0VmxExitMtf;
363static FNVMXEXITHANDLER hmR0VmxExitMonitor;
364static FNVMXEXITHANDLER hmR0VmxExitPause;
365static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
366static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
367static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
368static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
369static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
370static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
371static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
372static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
373static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
374static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
375static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
376static FNVMXEXITHANDLER hmR0VmxExitRdrand;
377static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
378/** @} */
379
380static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
381static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
382static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
383static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
384static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
385static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
386static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
387static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
388
389/*******************************************************************************
390* Global Variables *
391*******************************************************************************/
392#ifdef HMVMX_USE_FUNCTION_TABLE
393
394/**
395 * VMX_EXIT dispatch table.
396 */
397static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
398{
399 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
400 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
401 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
402 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
403 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
404 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
405 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
406 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
407 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
408 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
409 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
410 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
411 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
412 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
413 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
414 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
415 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
416 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
417 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
418 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
419 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
420 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
421 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
422 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
423 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
424 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
425 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
426 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
427 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
428 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
429 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
430 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
431 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
432 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
433 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
434 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
435 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
436 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
437 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
438 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
439 /* 40 UNDEFINED */ hmR0VmxExitPause,
440 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
441 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
442 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
443 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
444 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
445 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
446 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
447 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
448 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
449 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
450 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
451 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
452 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
453 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
454 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
455 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
456 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
457 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
458 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
459};
460#endif /* HMVMX_USE_FUNCTION_TABLE */
461
462#ifdef VBOX_STRICT
463static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
464{
465 /* 0 */ "(Not Used)",
466 /* 1 */ "VMCALL executed in VMX root operation.",
467 /* 2 */ "VMCLEAR with invalid physical address.",
468 /* 3 */ "VMCLEAR with VMXON pointer.",
469 /* 4 */ "VMLAUNCH with non-clear VMCS.",
470 /* 5 */ "VMRESUME with non-launched VMCS.",
471 /* 6 */ "VMRESUME after VMXOFF",
472 /* 7 */ "VM entry with invalid control fields.",
473 /* 8 */ "VM entry with invalid host state fields.",
474 /* 9 */ "VMPTRLD with invalid physical address.",
475 /* 10 */ "VMPTRLD with VMXON pointer.",
476 /* 11 */ "VMPTRLD with incorrect revision identifier.",
477 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
478 /* 13 */ "VMWRITE to read-only VMCS component.",
479 /* 14 */ "(Not Used)",
480 /* 15 */ "VMXON executed in VMX root operation.",
481 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
482 /* 17 */ "VM entry with non-launched executing VMCS.",
483 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
484 /* 19 */ "VMCALL with non-clear VMCS.",
485 /* 20 */ "VMCALL with invalid VM-exit control fields.",
486 /* 21 */ "(Not Used)",
487 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
488 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
489 /* 24 */ "VMCALL with invalid SMM-monitor features.",
490 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
491 /* 26 */ "VM entry with events blocked by MOV SS.",
492 /* 27 */ "(Not Used)",
493 /* 28 */ "Invalid operand to INVEPT/INVVPID."
494};
495#endif /* VBOX_STRICT */
496
497
498
499/**
500 * Updates the VM's last error record. If there was a VMX instruction error,
501 * reads the error data from the VMCS and updates VCPU's last error record as
502 * well.
503 *
504 * @param pVM Pointer to the VM.
505 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
506 * VERR_VMX_UNABLE_TO_START_VM or
507 * VERR_VMX_INVALID_VMCS_FIELD).
508 * @param rc The error code.
509 */
510static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
511{
512 AssertPtr(pVM);
513 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
514 || rc == VERR_VMX_UNABLE_TO_START_VM)
515 {
516 AssertPtrReturnVoid(pVCpu);
517 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
518 }
519 pVM->hm.s.lLastError = rc;
520}
521
522
523/**
524 * Reads the VM-entry interruption-information field from the VMCS into the VMX
525 * transient structure.
526 *
527 * @returns VBox status code.
528 * @param pVmxTransient Pointer to the VMX transient structure.
529 *
530 * @remarks No-long-jump zone!!!
531 */
532DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
533{
534 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
535 AssertRCReturn(rc, rc);
536 return VINF_SUCCESS;
537}
538
539
540/**
541 * Reads the VM-entry exception error code field from the VMCS into
542 * the VMX transient structure.
543 *
544 * @returns VBox status code.
545 * @param pVmxTransient Pointer to the VMX transient structure.
546 *
547 * @remarks No-long-jump zone!!!
548 */
549DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
550{
551 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
552 AssertRCReturn(rc, rc);
553 return VINF_SUCCESS;
554}
555
556
557/**
558 * Reads the VM-entry exception error code field from the VMCS into
559 * the VMX transient structure.
560 *
561 * @returns VBox status code.
562 * @param pVCpu Pointer to the VMCPU.
563 * @param pVmxTransient Pointer to the VMX transient structure.
564 *
565 * @remarks No-long-jump zone!!!
566 */
567DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
568{
569 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
570 AssertRCReturn(rc, rc);
571 return VINF_SUCCESS;
572}
573
574
575/**
576 * Reads the VM-exit interruption-information field from the VMCS into the VMX
577 * transient structure.
578 *
579 * @returns VBox status code.
580 * @param pVCpu Pointer to the VMCPU.
581 * @param pVmxTransient Pointer to the VMX transient structure.
582 */
583DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
584{
585 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
586 {
587 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
588 AssertRCReturn(rc, rc);
589 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
590 }
591 return VINF_SUCCESS;
592}
593
594
595/**
596 * Reads the VM-exit interruption error code from the VMCS into the VMX
597 * transient structure.
598 *
599 * @returns VBox status code.
600 * @param pVCpu Pointer to the VMCPU.
601 * @param pVmxTransient Pointer to the VMX transient structure.
602 */
603DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
604{
605 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
606 {
607 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
608 AssertRCReturn(rc, rc);
609 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
610 }
611 return VINF_SUCCESS;
612}
613
614
615/**
616 * Reads the VM-exit instruction length field from the VMCS into the VMX
617 * transient structure.
618 *
619 * @returns VBox status code.
620 * @param pVCpu Pointer to the VMCPU.
621 * @param pVmxTransient Pointer to the VMX transient structure.
622 */
623DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
624{
625 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
626 {
627 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
628 AssertRCReturn(rc, rc);
629 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
630 }
631 return VINF_SUCCESS;
632}
633
634
635/**
636 * Reads the VM-exit instruction-information field from the VMCS into
637 * the VMX transient structure.
638 *
639 * @returns VBox status code.
640 * @param pVCpu The cross context per CPU structure.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
648 AssertRCReturn(rc, rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the exit qualification from the VMCS into the VMX transient structure.
657 *
658 * @returns VBox status code.
659 * @param pVCpu Pointer to the VMCPU.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
665 {
666 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the IDT-vectoring information field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 *
681 * @remarks No-long-jump zone!!!
682 */
683DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
684{
685 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
686 {
687 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
688 AssertRCReturn(rc, rc);
689 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
690 }
691 return VINF_SUCCESS;
692}
693
694
695/**
696 * Reads the IDT-vectoring error code from the VMCS into the VMX
697 * transient structure.
698 *
699 * @returns VBox status code.
700 * @param pVmxTransient Pointer to the VMX transient structure.
701 */
702DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
703{
704 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
705 {
706 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
707 AssertRCReturn(rc, rc);
708 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
709 }
710 return VINF_SUCCESS;
711}
712
713
714/**
715 * Enters VMX root mode operation on the current CPU.
716 *
717 * @returns VBox status code.
718 * @param pVM Pointer to the VM (optional, can be NULL, after
719 * a resume).
720 * @param HCPhysCpuPage Physical address of the VMXON region.
721 * @param pvCpuPage Pointer to the VMXON region.
722 */
723static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
724{
725 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
726 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
727 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
728
729 if (pVM)
730 {
731 /* Write the VMCS revision dword to the VMXON region. */
732 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
733 }
734
735 /* Enable the VMX bit in CR4 if necessary. */
736 RTCCUINTREG uCr4 = ASMGetCR4();
737 if (!(uCr4 & X86_CR4_VMXE))
738 ASMSetCR4(uCr4 | X86_CR4_VMXE);
739
740 /* Enter VMX root mode. */
741 int rc = VMXEnable(HCPhysCpuPage);
742 if (RT_FAILURE(rc))
743 ASMSetCR4(uCr4);
744
745 return rc;
746}
747
748
749/**
750 * Exits VMX root mode operation on the current CPU.
751 *
752 * @returns VBox status code.
753 */
754static int hmR0VmxLeaveRootMode(void)
755{
756 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
757
758 /* If we're for some reason not in VMX root mode, then don't leave it. */
759 RTCCUINTREG uHostCR4 = ASMGetCR4();
760 if (uHostCR4 & X86_CR4_VMXE)
761 {
762 /* Exit VMX root mode and clear the VMX bit in CR4. */
763 VMXDisable();
764 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
765 return VINF_SUCCESS;
766 }
767
768 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
769}
770
771
772/**
773 * Allocates and maps one physically contiguous page. The allocated page is
774 * zero'd out. (Used by various VT-x structures).
775 *
776 * @returns IPRT status code.
777 * @param pMemObj Pointer to the ring-0 memory object.
778 * @param ppVirt Where to store the virtual address of the
779 * allocation.
780 * @param pPhys Where to store the physical address of the
781 * allocation.
782 */
783DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
784{
785 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
786 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
787 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
788
789 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
790 if (RT_FAILURE(rc))
791 return rc;
792 *ppVirt = RTR0MemObjAddress(*pMemObj);
793 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
794 ASMMemZero32(*ppVirt, PAGE_SIZE);
795 return VINF_SUCCESS;
796}
797
798
799/**
800 * Frees and unmaps an allocated physical page.
801 *
802 * @param pMemObj Pointer to the ring-0 memory object.
803 * @param ppVirt Where to re-initialize the virtual address of
804 * allocation as 0.
805 * @param pHCPhys Where to re-initialize the physical address of the
806 * allocation as 0.
807 */
808DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
809{
810 AssertPtr(pMemObj);
811 AssertPtr(ppVirt);
812 AssertPtr(pHCPhys);
813 if (*pMemObj != NIL_RTR0MEMOBJ)
814 {
815 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
816 AssertRC(rc);
817 *pMemObj = NIL_RTR0MEMOBJ;
818 *ppVirt = 0;
819 *pHCPhys = 0;
820 }
821}
822
823
824/**
825 * Worker function to free VT-x related structures.
826 *
827 * @returns IPRT status code.
828 * @param pVM Pointer to the VM.
829 */
830static void hmR0VmxStructsFree(PVM pVM)
831{
832 for (VMCPUID i = 0; i < pVM->cCpus; i++)
833 {
834 PVMCPU pVCpu = &pVM->aCpus[i];
835 AssertPtr(pVCpu);
836
837#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
838 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
839 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
840#endif
841
842 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
843 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
844
845 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
846 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
847 }
848
849 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
850#ifdef VBOX_WITH_CRASHDUMP_MAGIC
851 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
852#endif
853}
854
855
856/**
857 * Worker function to allocate VT-x related VM structures.
858 *
859 * @returns IPRT status code.
860 * @param pVM Pointer to the VM.
861 */
862static int hmR0VmxStructsAlloc(PVM pVM)
863{
864 /*
865 * Initialize members up-front so we can cleanup properly on allocation failure.
866 */
867#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
868 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
869 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
870 pVM->hm.s.vmx.HCPhys##a_Name = 0;
871
872#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
873 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
874 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
875 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
876
877#ifdef VBOX_WITH_CRASHDUMP_MAGIC
878 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
879#endif
880 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
881
882 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
883 for (VMCPUID i = 0; i < pVM->cCpus; i++)
884 {
885 PVMCPU pVCpu = &pVM->aCpus[i];
886 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
887 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
888 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
889#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
890 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
891 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
892#endif
893 }
894#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
895#undef VMXLOCAL_INIT_VM_MEMOBJ
896
897 /*
898 * Allocate all the VT-x structures.
899 */
900 int rc = VINF_SUCCESS;
901#ifdef VBOX_WITH_CRASHDUMP_MAGIC
902 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
903 if (RT_FAILURE(rc))
904 goto cleanup;
905 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
906 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
907#endif
908
909 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
910 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
911 {
912 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
913 &pVM->hm.s.vmx.HCPhysApicAccess);
914 if (RT_FAILURE(rc))
915 goto cleanup;
916 }
917
918 /*
919 * Initialize per-VCPU VT-x structures.
920 */
921 for (VMCPUID i = 0; i < pVM->cCpus; i++)
922 {
923 PVMCPU pVCpu = &pVM->aCpus[i];
924 AssertPtr(pVCpu);
925
926 /* Allocate the VM control structure (VMCS). */
927 AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
928 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
929 if (RT_FAILURE(rc))
930 goto cleanup;
931
932 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
933 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
934 {
935 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
936 &pVCpu->hm.s.vmx.HCPhysVirtApic);
937 if (RT_FAILURE(rc))
938 goto cleanup;
939 }
940
941 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
942 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
943 {
944 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
945 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
946 if (RT_FAILURE(rc))
947 goto cleanup;
948 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
949 }
950
951#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
952 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
953 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956
957 /* Allocate the VM-exit MSR-load page for the host MSRs. */
958 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
959 if (RT_FAILURE(rc))
960 goto cleanup;
961#endif
962 }
963
964 return VINF_SUCCESS;
965
966cleanup:
967 hmR0VmxStructsFree(pVM);
968 return rc;
969}
970
971
972/**
973 * Does global VT-x initialization (called during module initialization).
974 *
975 * @returns VBox status code.
976 */
977VMMR0DECL(int) VMXR0GlobalInit(void)
978{
979#ifdef HMVMX_USE_FUNCTION_TABLE
980 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
981# ifdef VBOX_STRICT
982 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
983 Assert(g_apfnVMExitHandlers[i]);
984# endif
985#endif
986 return VINF_SUCCESS;
987}
988
989
990/**
991 * Does global VT-x termination (called during module termination).
992 */
993VMMR0DECL(void) VMXR0GlobalTerm()
994{
995 /* Nothing to do currently. */
996}
997
998
999/**
1000 * Sets up and activates VT-x on the current CPU.
1001 *
1002 * @returns VBox status code.
1003 * @param pCpu Pointer to the global CPU info struct.
1004 * @param pVM Pointer to the VM (can be NULL after a host resume
1005 * operation).
1006 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1007 * fEnabledByHost is true).
1008 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1009 * @a fEnabledByHost is true).
1010 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1011 * enable VT-x on the host.
1012 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1013 */
1014VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1015 void *pvMsrs)
1016{
1017 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1018 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1019 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1020
1021 /* Enable VT-x if it's not already enabled by the host. */
1022 if (!fEnabledByHost)
1023 {
1024 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1025 if (RT_FAILURE(rc))
1026 return rc;
1027 }
1028
1029 /*
1030 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1031 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1032 */
1033 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1034 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1035 {
1036 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1037 pCpu->fFlushAsidBeforeUse = false;
1038 }
1039 else
1040 pCpu->fFlushAsidBeforeUse = true;
1041
1042 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1043 ++pCpu->cTlbFlushes;
1044
1045 return VINF_SUCCESS;
1046}
1047
1048
1049/**
1050 * Deactivates VT-x on the current CPU.
1051 *
1052 * @returns VBox status code.
1053 * @param pCpu Pointer to the global CPU info struct.
1054 * @param pvCpuPage Pointer to the VMXON region.
1055 * @param HCPhysCpuPage Physical address of the VMXON region.
1056 *
1057 * @remarks This function should never be called when SUPR0EnableVTx() or
1058 * similar was used to enable VT-x on the host.
1059 */
1060VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1061{
1062 NOREF(pCpu);
1063 NOREF(pvCpuPage);
1064 NOREF(HCPhysCpuPage);
1065
1066 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1067 return hmR0VmxLeaveRootMode();
1068}
1069
1070
1071/**
1072 * Sets the permission bits for the specified MSR in the MSR bitmap.
1073 *
1074 * @param pVCpu Pointer to the VMCPU.
1075 * @param uMSR The MSR value.
1076 * @param enmRead Whether reading this MSR causes a VM-exit.
1077 * @param enmWrite Whether writing this MSR causes a VM-exit.
1078 */
1079static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1080{
1081 int32_t iBit;
1082 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1083
1084 /*
1085 * Layout:
1086 * 0x000 - 0x3ff - Low MSR read bits
1087 * 0x400 - 0x7ff - High MSR read bits
1088 * 0x800 - 0xbff - Low MSR write bits
1089 * 0xc00 - 0xfff - High MSR write bits
1090 */
1091 if (uMsr <= 0x00001FFF)
1092 iBit = uMsr;
1093 else if ( uMsr >= 0xC0000000
1094 && uMsr <= 0xC0001FFF)
1095 {
1096 iBit = (uMsr - 0xC0000000);
1097 pbMsrBitmap += 0x400;
1098 }
1099 else
1100 {
1101 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1102 return;
1103 }
1104
1105 Assert(iBit <= 0x1fff);
1106 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1107 ASMBitSet(pbMsrBitmap, iBit);
1108 else
1109 ASMBitClear(pbMsrBitmap, iBit);
1110
1111 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1112 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1113 else
1114 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1115}
1116
1117
1118/**
1119 * Flushes the TLB using EPT.
1120 *
1121 * @returns VBox status code.
1122 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1123 * enmFlush).
1124 * @param enmFlush Type of flush.
1125 *
1126 * @remarks Caller is responsible for making sure this function is called only
1127 * when NestedPaging is supported and providing @a enmFlush that is
1128 * supported by the CPU.
1129 */
1130static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1131{
1132 uint64_t au64Descriptor[2];
1133 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1134 au64Descriptor[0] = 0;
1135 else
1136 {
1137 Assert(pVCpu);
1138 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1139 }
1140 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1141
1142 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1143 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1144 rc));
1145 if ( RT_SUCCESS(rc)
1146 && pVCpu)
1147 {
1148 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1149 }
1150}
1151
1152
1153/**
1154 * Flushes the TLB using VPID.
1155 *
1156 * @returns VBox status code.
1157 * @param pVM Pointer to the VM.
1158 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1159 * enmFlush).
1160 * @param enmFlush Type of flush.
1161 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1162 * on @a enmFlush).
1163 */
1164static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1165{
1166 AssertPtr(pVM);
1167 Assert(pVM->hm.s.vmx.fVpid);
1168
1169 uint64_t au64Descriptor[2];
1170 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1171 {
1172 au64Descriptor[0] = 0;
1173 au64Descriptor[1] = 0;
1174 }
1175 else
1176 {
1177 AssertPtr(pVCpu);
1178 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1179 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1180 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1181 au64Descriptor[1] = GCPtr;
1182 }
1183
1184 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1185 AssertMsg(rc == VINF_SUCCESS,
1186 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1187 if ( RT_SUCCESS(rc)
1188 && pVCpu)
1189 {
1190 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1191 }
1192}
1193
1194
1195/**
1196 * Invalidates a guest page by guest virtual address. Only relevant for
1197 * EPT/VPID, otherwise there is nothing really to invalidate.
1198 *
1199 * @returns VBox status code.
1200 * @param pVM Pointer to the VM.
1201 * @param pVCpu Pointer to the VMCPU.
1202 * @param GCVirt Guest virtual address of the page to invalidate.
1203 */
1204VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1205{
1206 AssertPtr(pVM);
1207 AssertPtr(pVCpu);
1208 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1209
1210 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1211 if (!fFlushPending)
1212 {
1213 /*
1214 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1215 * See @bugref{6043} and @bugref{6177}.
1216 *
1217 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1218 * function maybe called in a loop with individual addresses.
1219 */
1220 if (pVM->hm.s.vmx.fVpid)
1221 {
1222 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1223 {
1224 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1225 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1226 }
1227 else
1228 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1229 }
1230 else if (pVM->hm.s.fNestedPaging)
1231 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1232 }
1233
1234 return VINF_SUCCESS;
1235}
1236
1237
1238/**
1239 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1240 * otherwise there is nothing really to invalidate.
1241 *
1242 * @returns VBox status code.
1243 * @param pVM Pointer to the VM.
1244 * @param pVCpu Pointer to the VMCPU.
1245 * @param GCPhys Guest physical address of the page to invalidate.
1246 */
1247VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1248{
1249 LogFlowFunc(("%RGp\n", GCPhys));
1250
1251 /*
1252 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1253 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1254 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1255 */
1256 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1257 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1258 return VINF_SUCCESS;
1259}
1260
1261
1262/**
1263 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1264 * case where neither EPT nor VPID is supported by the CPU.
1265 *
1266 * @param pVM Pointer to the VM.
1267 * @param pVCpu Pointer to the VMCPU.
1268 * @param pCpu Pointer to the global HM struct.
1269 *
1270 * @remarks Called with interrupts disabled.
1271 */
1272static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1273{
1274 AssertPtr(pVCpu);
1275 AssertPtr(pCpu);
1276 NOREF(pVM);
1277
1278 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1279 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1280
1281 pVCpu->hm.s.TlbShootdown.cPages = 0;
1282 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1283 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1284 pVCpu->hm.s.fForceTLBFlush = false;
1285 return;
1286}
1287
1288
1289/**
1290 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1291 *
1292 * @param pVM Pointer to the VM.
1293 * @param pVCpu Pointer to the VMCPU.
1294 * @param pCpu Pointer to the global HM CPU struct.
1295 * @remarks All references to "ASID" in this function pertains to "VPID" in
1296 * Intel's nomenclature. The reason is, to avoid confusion in compare
1297 * statements since the host-CPU copies are named "ASID".
1298 *
1299 * @remarks Called with interrupts disabled.
1300 */
1301static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1302{
1303#ifdef VBOX_WITH_STATISTICS
1304 bool fTlbFlushed = false;
1305# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1306# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1307 if (!fTlbFlushed) \
1308 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1309 } while (0)
1310#else
1311# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1312# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1313#endif
1314
1315 AssertPtr(pVM);
1316 AssertPtr(pCpu);
1317 AssertPtr(pVCpu);
1318 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1319 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1320 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1321
1322
1323 /*
1324 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1325 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1326 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1327 */
1328 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1329 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1330 {
1331 ++pCpu->uCurrentAsid;
1332 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1333 {
1334 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1335 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1336 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1337 }
1338
1339 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1340 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1341 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1342
1343 /*
1344 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1345 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1346 */
1347 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1348 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1349 HMVMX_SET_TAGGED_TLB_FLUSHED();
1350 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1351 }
1352
1353 /* Check for explicit TLB shootdowns. */
1354 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1355 {
1356 /*
1357 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1358 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1359 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1360 * but not guest-physical mappings.
1361 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1362 */
1363 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1364 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1365 HMVMX_SET_TAGGED_TLB_FLUSHED();
1366 }
1367
1368 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1369 * not be executed. See hmQueueInvlPage() where it is commented
1370 * out. Support individual entry flushing someday. */
1371 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1372 {
1373 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1374
1375 /*
1376 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1377 * as supported by the CPU.
1378 */
1379 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1380 {
1381 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1382 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1383 }
1384 else
1385 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1386
1387 HMVMX_SET_TAGGED_TLB_FLUSHED();
1388 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1389 }
1390
1391 pVCpu->hm.s.TlbShootdown.cPages = 0;
1392 pVCpu->hm.s.fForceTLBFlush = false;
1393
1394 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1395
1396 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1397 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1398 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1399 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1400 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1401 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1402 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1403 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1404
1405 /* Update VMCS with the VPID. */
1406 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1407 AssertRC(rc);
1408
1409#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1410}
1411
1412
1413/**
1414 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1415 *
1416 * @returns VBox status code.
1417 * @param pVM Pointer to the VM.
1418 * @param pVCpu Pointer to the VMCPU.
1419 * @param pCpu Pointer to the global HM CPU struct.
1420 *
1421 * @remarks Called with interrupts disabled.
1422 */
1423static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1424{
1425 AssertPtr(pVM);
1426 AssertPtr(pVCpu);
1427 AssertPtr(pCpu);
1428 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1429 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1430
1431 /*
1432 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1433 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1434 */
1435 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1436 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1437 {
1438 pVCpu->hm.s.fForceTLBFlush = true;
1439 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1440 }
1441
1442 /* Check for explicit TLB shootdown flushes. */
1443 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1444 {
1445 pVCpu->hm.s.fForceTLBFlush = true;
1446 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1447 }
1448
1449 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1450 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1451
1452 if (pVCpu->hm.s.fForceTLBFlush)
1453 {
1454 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1455 pVCpu->hm.s.fForceTLBFlush = false;
1456 }
1457 else
1458 {
1459 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1460 * not be executed. See hmQueueInvlPage() where it is commented
1461 * out. Support individual entry flushing someday. */
1462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1463 {
1464 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1465 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1466 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1467 }
1468 else
1469 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1470 }
1471
1472 pVCpu->hm.s.TlbShootdown.cPages = 0;
1473 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1474}
1475
1476
1477/**
1478 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1479 *
1480 * @returns VBox status code.
1481 * @param pVM Pointer to the VM.
1482 * @param pVCpu Pointer to the VMCPU.
1483 * @param pCpu Pointer to the global HM CPU struct.
1484 *
1485 * @remarks Called with interrupts disabled.
1486 */
1487static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1488{
1489 AssertPtr(pVM);
1490 AssertPtr(pVCpu);
1491 AssertPtr(pCpu);
1492 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1493 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1494
1495 /*
1496 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1497 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1498 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1499 */
1500 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1501 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1502 {
1503 pVCpu->hm.s.fForceTLBFlush = true;
1504 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1505 }
1506
1507 /* Check for explicit TLB shootdown flushes. */
1508 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1509 {
1510 /*
1511 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1512 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1513 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1514 */
1515 pVCpu->hm.s.fForceTLBFlush = true;
1516 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1517 }
1518
1519 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1520 if (pVCpu->hm.s.fForceTLBFlush)
1521 {
1522 ++pCpu->uCurrentAsid;
1523 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1524 {
1525 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1526 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1527 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1528 }
1529
1530 pVCpu->hm.s.fForceTLBFlush = false;
1531 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1532 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1533 if (pCpu->fFlushAsidBeforeUse)
1534 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1535 }
1536 else
1537 {
1538 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1539 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1540 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1541 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1542
1543 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1544 * not be executed. See hmQueueInvlPage() where it is commented
1545 * out. Support individual entry flushing someday. */
1546 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1547 {
1548 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1549 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1550 {
1551 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1552 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1553 }
1554 else
1555 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1556 }
1557 else
1558 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1559 }
1560
1561 pVCpu->hm.s.TlbShootdown.cPages = 0;
1562 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1563
1564 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1565 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1566 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1567 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1568 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1569 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1570
1571 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1572 AssertRC(rc);
1573}
1574
1575
1576/**
1577 * Flushes the guest TLB entry based on CPU capabilities.
1578 *
1579 * @param pVCpu Pointer to the VMCPU.
1580 * @param pCpu Pointer to the global HM CPU struct.
1581 */
1582DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1583{
1584 PVM pVM = pVCpu->CTX_SUFF(pVM);
1585 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1586 {
1587 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1588 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1589 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1590 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1591 default:
1592 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1593 break;
1594 }
1595}
1596
1597
1598/**
1599 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1600 * TLB entries from the host TLB before VM-entry.
1601 *
1602 * @returns VBox status code.
1603 * @param pVM Pointer to the VM.
1604 */
1605static int hmR0VmxSetupTaggedTlb(PVM pVM)
1606{
1607 /*
1608 * Determine optimal flush type for Nested Paging.
1609 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1610 * guest execution (see hmR3InitFinalizeR0()).
1611 */
1612 if (pVM->hm.s.fNestedPaging)
1613 {
1614 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1615 {
1616 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1617 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1618 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1619 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1620 else
1621 {
1622 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1623 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1624 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1625 }
1626
1627 /* Make sure the write-back cacheable memory type for EPT is supported. */
1628 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1629 {
1630 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1631 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1632 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1633 }
1634 }
1635 else
1636 {
1637 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1638 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1639 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1640 }
1641 }
1642
1643 /*
1644 * Determine optimal flush type for VPID.
1645 */
1646 if (pVM->hm.s.vmx.fVpid)
1647 {
1648 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1649 {
1650 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1651 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1652 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1653 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1654 else
1655 {
1656 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1657 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1658 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1659 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1660 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1661 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1662 pVM->hm.s.vmx.fVpid = false;
1663 }
1664 }
1665 else
1666 {
1667 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1668 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1669 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1670 pVM->hm.s.vmx.fVpid = false;
1671 }
1672 }
1673
1674 /*
1675 * Setup the handler for flushing tagged-TLBs.
1676 */
1677 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1678 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1679 else if (pVM->hm.s.fNestedPaging)
1680 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1681 else if (pVM->hm.s.vmx.fVpid)
1682 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1683 else
1684 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1685 return VINF_SUCCESS;
1686}
1687
1688
1689/**
1690 * Sets up pin-based VM-execution controls in the VMCS.
1691 *
1692 * @returns VBox status code.
1693 * @param pVM Pointer to the VM.
1694 * @param pVCpu Pointer to the VMCPU.
1695 */
1696static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1697{
1698 AssertPtr(pVM);
1699 AssertPtr(pVCpu);
1700
1701 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1702 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1703
1704 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1705 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1706 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1707
1708 /* Enable the VMX preemption timer. */
1709 if (pVM->hm.s.vmx.fUsePreemptTimer)
1710 {
1711 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1712 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1713 }
1714
1715 if ((val & zap) != val)
1716 {
1717 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1718 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1719 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1720 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1721 }
1722
1723 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1724 AssertRCReturn(rc, rc);
1725
1726 /* Update VCPU with the currently set pin-based VM-execution controls. */
1727 pVCpu->hm.s.vmx.u32PinCtls = val;
1728 return rc;
1729}
1730
1731
1732/**
1733 * Sets up processor-based VM-execution controls in the VMCS.
1734 *
1735 * @returns VBox status code.
1736 * @param pVM Pointer to the VM.
1737 * @param pVMCPU Pointer to the VMCPU.
1738 */
1739static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1740{
1741 AssertPtr(pVM);
1742 AssertPtr(pVCpu);
1743
1744 int rc = VERR_INTERNAL_ERROR_5;
1745 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1746 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1747
1748 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1749 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1750 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1751 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1752 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1753 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1754 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1755
1756 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1757 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1758 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1759 {
1760 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1761 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1762 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1763 }
1764
1765 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1766 if (!pVM->hm.s.fNestedPaging)
1767 {
1768 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1769 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1770 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1771 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1772 }
1773
1774 /* Use TPR shadowing if supported by the CPU. */
1775 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1776 {
1777 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1778 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1779 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1780 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1781 AssertRCReturn(rc, rc);
1782
1783 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1784 /* CR8 writes causes a VM-exit based on TPR threshold. */
1785 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1786 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1787 }
1788 else
1789 {
1790 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1791 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1792 }
1793
1794 /* Use MSR-bitmaps if supported by the CPU. */
1795 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1796 {
1797 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1798
1799 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1800 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1801 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1802 AssertRCReturn(rc, rc);
1803
1804 /*
1805 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1806 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1807 */
1808 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1809 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1810 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1811 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1812 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1813 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1814 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1815 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1816 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1817 }
1818
1819 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1820 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1821 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1822
1823 if ((val & zap) != val)
1824 {
1825 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1826 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1827 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1828 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1829 }
1830
1831 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1832 AssertRCReturn(rc, rc);
1833
1834 /* Update VCPU with the currently set processor-based VM-execution controls. */
1835 pVCpu->hm.s.vmx.u32ProcCtls = val;
1836
1837 /*
1838 * Secondary processor-based VM-execution controls.
1839 */
1840 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1841 {
1842 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1843 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1844
1845 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1846 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1847
1848 if (pVM->hm.s.fNestedPaging)
1849 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1850 else
1851 {
1852 /*
1853 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1854 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1855 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1856 */
1857 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1858 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1859 }
1860
1861 if (pVM->hm.s.vmx.fVpid)
1862 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1863
1864 if (pVM->hm.s.vmx.fUnrestrictedGuest)
1865 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1866
1867 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1868 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1869 * done dynamically. */
1870 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1871 {
1872 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1873 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1874 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1875 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1876 AssertRCReturn(rc, rc);
1877 }
1878
1879 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1880 {
1881 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1882 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1883 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1884 }
1885
1886 if ((val & zap) != val)
1887 {
1888 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1889 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1890 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1891 }
1892
1893 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1894 AssertRCReturn(rc, rc);
1895
1896 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1897 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1898 }
1899 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1900 {
1901 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1902 "available\n"));
1903 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1904 }
1905
1906 return VINF_SUCCESS;
1907}
1908
1909
1910/**
1911 * Sets up miscellaneous (everything other than Pin & Processor-based
1912 * VM-execution) control fields in the VMCS.
1913 *
1914 * @returns VBox status code.
1915 * @param pVM Pointer to the VM.
1916 * @param pVCpu Pointer to the VMCPU.
1917 */
1918static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1919{
1920 AssertPtr(pVM);
1921 AssertPtr(pVCpu);
1922
1923 int rc = VERR_GENERAL_FAILURE;
1924
1925 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1926#if 0
1927 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1928 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1929 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1930
1931 /*
1932 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1933 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1934 * We thus use the exception bitmap to control it rather than use both.
1935 */
1936 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1937 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1938
1939 /** @todo Explore possibility of using IO-bitmaps. */
1940 /* All IO & IOIO instructions cause VM-exits. */
1941 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1942 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1943
1944 /* Initialize the MSR-bitmap area. */
1945 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1946 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1947 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1948#endif
1949
1950#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1951 /* Setup MSR autoloading/storing. */
1952 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1953 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1954 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1955 AssertRCReturn(rc, rc);
1956 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1957 AssertRCReturn(rc, rc);
1958
1959 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1960 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1961 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1962 AssertRCReturn(rc, rc);
1963#endif
1964
1965 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1966 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1967 AssertRCReturn(rc, rc);
1968
1969 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1970#if 0
1971 /* Setup debug controls */
1972 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1973 AssertRCReturn(rc, rc);
1974 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1975 AssertRCReturn(rc, rc);
1976#endif
1977
1978 return rc;
1979}
1980
1981
1982/**
1983 * Sets up the initial exception bitmap in the VMCS based on static conditions
1984 * (i.e. conditions that cannot ever change after starting the VM).
1985 *
1986 * @returns VBox status code.
1987 * @param pVM Pointer to the VM.
1988 * @param pVCpu Pointer to the VMCPU.
1989 */
1990static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1991{
1992 AssertPtr(pVM);
1993 AssertPtr(pVCpu);
1994
1995 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1996
1997 uint32_t u32XcptBitmap = 0;
1998
1999 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2000 if (!pVM->hm.s.fNestedPaging)
2001 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2002
2003 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2004 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2005 AssertRCReturn(rc, rc);
2006 return rc;
2007}
2008
2009
2010/**
2011 * Sets up the initial guest-state mask. The guest-state mask is consulted
2012 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2013 * for the nested virtualization case (as it would cause a VM-exit).
2014 *
2015 * @param pVCpu Pointer to the VMCPU.
2016 */
2017static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2018{
2019 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2020 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2021 return VINF_SUCCESS;
2022}
2023
2024
2025/**
2026 * Does per-VM VT-x initialization.
2027 *
2028 * @returns VBox status code.
2029 * @param pVM Pointer to the VM.
2030 */
2031VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2032{
2033 LogFlowFunc(("pVM=%p\n", pVM));
2034
2035 int rc = hmR0VmxStructsAlloc(pVM);
2036 if (RT_FAILURE(rc))
2037 {
2038 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2039 return rc;
2040 }
2041
2042 return VINF_SUCCESS;
2043}
2044
2045
2046/**
2047 * Does per-VM VT-x termination.
2048 *
2049 * @returns VBox status code.
2050 * @param pVM Pointer to the VM.
2051 */
2052VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2053{
2054 LogFlowFunc(("pVM=%p\n", pVM));
2055
2056#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2057 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2058 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2059#endif
2060 hmR0VmxStructsFree(pVM);
2061 return VINF_SUCCESS;
2062}
2063
2064
2065/**
2066 * Sets up the VM for execution under VT-x.
2067 * This function is only called once per-VM during initialization.
2068 *
2069 * @returns VBox status code.
2070 * @param pVM Pointer to the VM.
2071 */
2072VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2073{
2074 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2075 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2076
2077 LogFlowFunc(("pVM=%p\n", pVM));
2078
2079 /*
2080 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2081 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2082 */
2083 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2084 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2085 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2086 || !pVM->hm.s.vmx.pRealModeTSS))
2087 {
2088 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2089 return VERR_INTERNAL_ERROR;
2090 }
2091
2092#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2093 /*
2094 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2095 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2096 */
2097 if ( pVM->hm.s.fAllow64BitGuests
2098 && !HMVMX_IS_64BIT_HOST_MODE())
2099 {
2100 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2101 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2102 }
2103#endif
2104
2105 /* Initialize these always, see hmR3InitFinalizeR0().*/
2106 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2107 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2108
2109 /* Setup the tagged-TLB flush handlers. */
2110 int rc = hmR0VmxSetupTaggedTlb(pVM);
2111 if (RT_FAILURE(rc))
2112 {
2113 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2114 return rc;
2115 }
2116
2117 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2118 {
2119 PVMCPU pVCpu = &pVM->aCpus[i];
2120 AssertPtr(pVCpu);
2121 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2122
2123 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2124 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2125
2126 /* Set revision dword at the beginning of the VMCS structure. */
2127 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2128
2129 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2130 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2131 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2132 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2133
2134 /* Load this VMCS as the current VMCS. */
2135 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2136 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2137 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2138
2139 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2140 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2141 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2142
2143 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2144 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2145 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2146
2147 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2148 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2149 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2150
2151 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2152 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2153 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2154
2155 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2156 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2157 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2158
2159#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2160 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2161 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2162 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2163#endif
2164
2165 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2166 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2167 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2168 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2169
2170 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2171
2172 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2173 }
2174
2175 return VINF_SUCCESS;
2176}
2177
2178
2179/**
2180 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2181 * the VMCS.
2182 *
2183 * @returns VBox status code.
2184 * @param pVM Pointer to the VM.
2185 * @param pVCpu Pointer to the VMCPU.
2186 */
2187DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2188{
2189 RTCCUINTREG uReg = ASMGetCR0();
2190 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2191 AssertRCReturn(rc, rc);
2192
2193#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2194 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2195 if (HMVMX_IS_64BIT_HOST_MODE())
2196 {
2197 uint64_t uRegCR3 = HMR0Get64bitCR3();
2198 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2199 }
2200 else
2201#endif
2202 {
2203 uReg = ASMGetCR3();
2204 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2205 }
2206 AssertRCReturn(rc, rc);
2207
2208 uReg = ASMGetCR4();
2209 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2210 AssertRCReturn(rc, rc);
2211 return rc;
2212}
2213
2214
2215/**
2216 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2217 * the host-state area in the VMCS.
2218 *
2219 * @returns VBox status code.
2220 * @param pVM Pointer to the VM.
2221 * @param pVCpu Pointer to the VMCPU.
2222 */
2223DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2224{
2225 int rc = VERR_INTERNAL_ERROR_5;
2226
2227 /*
2228 * Host DS, ES, FS and GS segment registers.
2229 */
2230#if HC_ARCH_BITS == 64
2231 RTSEL uSelDS = ASMGetDS();
2232 RTSEL uSelES = ASMGetES();
2233 RTSEL uSelFS = ASMGetFS();
2234 RTSEL uSelGS = ASMGetGS();
2235#else
2236 RTSEL uSelDS = 0;
2237 RTSEL uSelES = 0;
2238 RTSEL uSelFS = 0;
2239 RTSEL uSelGS = 0;
2240#endif
2241
2242 /* Recalculate which host-state bits need to be manually restored. */
2243 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2244
2245 /*
2246 * Host CS and SS segment registers.
2247 */
2248#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2249 RTSEL uSelCS;
2250 RTSEL uSelSS;
2251 if (HMVMX_IS_64BIT_HOST_MODE())
2252 {
2253 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2254 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2255 }
2256 else
2257 {
2258 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2259 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2260 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2261 }
2262#else
2263 RTSEL uSelCS = ASMGetCS();
2264 RTSEL uSelSS = ASMGetSS();
2265#endif
2266
2267 /*
2268 * Host TR segment register.
2269 */
2270 RTSEL uSelTR = ASMGetTR();
2271
2272#if HC_ARCH_BITS == 64
2273 /*
2274 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2275 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2276 */
2277 if (uSelDS & (X86_SEL_RPL | X86_SEL_LDT))
2278 {
2279 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_DS;
2280 pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2281 uSelDS = 0;
2282 }
2283 if (uSelES & (X86_SEL_RPL | X86_SEL_LDT))
2284 {
2285 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_ES;
2286 pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2287 uSelES = 0;
2288 }
2289 if (uSelFS & (X86_SEL_RPL | X86_SEL_LDT))
2290 {
2291 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_FS;
2292 pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2293 uSelFS = 0;
2294 }
2295 if (uSelGS & (X86_SEL_RPL | X86_SEL_LDT))
2296 {
2297 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_GS;
2298 pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2299 uSelGS = 0;
2300 }
2301#endif
2302
2303 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2304 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2305 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2306 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2307 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2308 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2309 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2310 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2311 Assert(uSelCS);
2312 Assert(uSelTR);
2313
2314 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2315#if 0
2316 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2317 Assert(uSelSS != 0);
2318#endif
2319
2320 /* Write these host selector fields into the host-state area in the VMCS. */
2321 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2322 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2323#if HC_ARCH_BITS == 64
2324 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2325 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2326 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2327 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2328#endif
2329 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2330
2331 /*
2332 * Host GDTR and IDTR.
2333 */
2334 RTGDTR Gdtr;
2335 RT_ZERO(Gdtr);
2336#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2337 if (HMVMX_IS_64BIT_HOST_MODE())
2338 {
2339 X86XDTR64 Gdtr64;
2340 X86XDTR64 Idtr64;
2341 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2342 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2343 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2344
2345 Gdtr.cbGdt = Gdtr64.cb;
2346 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2347 }
2348 else
2349#endif
2350 {
2351 RTIDTR Idtr;
2352 ASMGetGDTR(&Gdtr);
2353 ASMGetIDTR(&Idtr);
2354 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2355 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2356
2357#if HC_ARCH_BITS == 64
2358 /*
2359 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2360 * maximum limit (0xffff) on every VM-exit.
2361 */
2362 if (Gdtr.cbGdt != 0xffff)
2363 {
2364 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2365 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2366 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2367 }
2368
2369 /*
2370 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2371 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2372 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2373 */
2374 if (Idtr.cbIdt < 0x0fff)
2375 {
2376 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2377 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2378 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2379 }
2380#endif
2381 }
2382
2383 /*
2384 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2385 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2386 */
2387 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2388 {
2389 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2390 return VERR_VMX_INVALID_HOST_STATE;
2391 }
2392
2393 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2394#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2395 if (HMVMX_IS_64BIT_HOST_MODE())
2396 {
2397 /* We need the 64-bit TR base for hybrid darwin. */
2398 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2399 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2400 }
2401 else
2402#endif
2403 {
2404 uintptr_t uTRBase;
2405#if HC_ARCH_BITS == 64
2406 uTRBase = X86DESC64_BASE(pDesc);
2407
2408 /*
2409 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2410 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2411 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2412 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2413 *
2414 * [1] See Intel spec. 3.5 "System Descriptor Types".
2415 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2416 */
2417 Assert(pDesc->System.u4Type == 11);
2418 if ( pDesc->System.u16LimitLow != 0x67
2419 || pDesc->System.u4LimitHigh)
2420 {
2421 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2422 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2423
2424 /* Store the GDTR here as we need it while restoring TR. */
2425 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2426 }
2427#else
2428 uTRBase = X86DESC_BASE(pDesc);
2429#endif
2430 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2431 }
2432 AssertRCReturn(rc, rc);
2433
2434 /*
2435 * Host FS base and GS base.
2436 */
2437#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2438 if (HMVMX_IS_64BIT_HOST_MODE())
2439 {
2440 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2441 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2442 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2443 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2444
2445# if HC_ARCH_BITS == 64
2446 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2447 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2448 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2449 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2450 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2451# endif
2452 }
2453#endif
2454 return rc;
2455}
2456
2457
2458/**
2459 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2460 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2461 * the host after every successful VM exit.
2462 *
2463 * @returns VBox status code.
2464 * @param pVM Pointer to the VM.
2465 * @param pVCpu Pointer to the VMCPU.
2466 */
2467DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2468{
2469 AssertPtr(pVCpu);
2470 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2471
2472 int rc = VINF_SUCCESS;
2473#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2474 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2475 uint32_t cHostMsrs = 0;
2476 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2477
2478 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2479 {
2480 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2481
2482# if HC_ARCH_BITS == 64
2483 /* Paranoia. 64-bit code requires these bits to be set always. */
2484 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2485
2486 /*
2487 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2488 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2489 * some reason (e.g. allow transparent reads) we would activate the code below.
2490 */
2491# if 0
2492 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2493 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2494 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2495 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2496 if (CPUMIsGuestInLongMode(pVCpu))
2497 {
2498 uint64_t u64GuestEfer;
2499 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2500 AssertRC(rc);
2501
2502 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2503 {
2504 pHostMsr->u32Msr = MSR_K6_EFER;
2505 pHostMsr->u32Reserved = 0;
2506 pHostMsr->u64Value = u64HostEfer;
2507 pHostMsr++; cHostMsrs++;
2508 }
2509 }
2510# endif
2511# else /* HC_ARCH_BITS != 64 */
2512 pHostMsr->u32Msr = MSR_K6_EFER;
2513 pHostMsr->u32Reserved = 0;
2514# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2515 if (CPUMIsGuestInLongMode(pVCpu))
2516 {
2517 /* Must match the EFER value in our 64 bits switcher. */
2518 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2519 }
2520 else
2521# endif
2522 pHostMsr->u64Value = u64HostEfer;
2523 pHostMsr++; cHostMsrs++;
2524# endif /* HC_ARCH_BITS == 64 */
2525 }
2526
2527# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2528 if (HMVMX_IS_64BIT_HOST_MODE())
2529 {
2530 pHostMsr->u32Msr = MSR_K6_STAR;
2531 pHostMsr->u32Reserved = 0;
2532 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2533 pHostMsr++; cHostMsrs++;
2534 pHostMsr->u32Msr = MSR_K8_LSTAR;
2535 pHostMsr->u32Reserved = 0;
2536 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2537 pHostMsr++; cHostMsrs++;
2538 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2539 pHostMsr->u32Reserved = 0;
2540 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2541 pHostMsr++; cHostMsrs++;
2542 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2543 pHostMsr->u32Reserved = 0;
2544 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2545 pHostMsr++; cHostMsrs++;
2546 }
2547# endif
2548
2549 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2550 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2551 {
2552 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2553 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2554 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2555 }
2556
2557 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2558#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2559
2560 /*
2561 * Host Sysenter MSRs.
2562 */
2563 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2564 AssertRCReturn(rc, rc);
2565#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2566 if (HMVMX_IS_64BIT_HOST_MODE())
2567 {
2568 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2569 AssertRCReturn(rc, rc);
2570 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2571 }
2572 else
2573 {
2574 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2575 AssertRCReturn(rc, rc);
2576 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2577 }
2578#elif HC_ARCH_BITS == 32
2579 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2580 AssertRCReturn(rc, rc);
2581 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2582#else
2583 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2584 AssertRCReturn(rc, rc);
2585 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2586#endif
2587 AssertRCReturn(rc, rc);
2588
2589 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2590 * hmR0VmxSetupExitCtls() !! */
2591 return rc;
2592}
2593
2594
2595/**
2596 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2597 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2598 * controls".
2599 *
2600 * @returns VBox status code.
2601 * @param pVCpu Pointer to the VMCPU.
2602 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2603 * out-of-sync. Make sure to update the required fields
2604 * before using them.
2605 *
2606 * @remarks No-long-jump zone!!!
2607 */
2608DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2609{
2610 int rc = VINF_SUCCESS;
2611 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2612 {
2613 PVM pVM = pVCpu->CTX_SUFF(pVM);
2614 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2615 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2616
2617 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2618 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2619
2620 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2621 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2622 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2623 else
2624 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2625
2626 /*
2627 * The following should -not- be set (since we're not in SMM mode):
2628 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2629 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2630 */
2631
2632 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2633 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2634 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2635
2636 if ((val & zap) != val)
2637 {
2638 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2639 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2640 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2641 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2642 }
2643
2644 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2645 AssertRCReturn(rc, rc);
2646
2647 /* Update VCPU with the currently set VM-exit controls. */
2648 pVCpu->hm.s.vmx.u32EntryCtls = val;
2649 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2650 }
2651 return rc;
2652}
2653
2654
2655/**
2656 * Sets up the VM-exit controls in the VMCS.
2657 *
2658 * @returns VBox status code.
2659 * @param pVM Pointer to the VM.
2660 * @param pVCpu Pointer to the VMCPU.
2661 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2662 * out-of-sync. Make sure to update the required fields
2663 * before using them.
2664 *
2665 * @remarks requires EFER.
2666 */
2667DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2668{
2669 int rc = VINF_SUCCESS;
2670 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2671 {
2672 PVM pVM = pVCpu->CTX_SUFF(pVM);
2673 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2674 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2675
2676 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2677 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2678
2679 /*
2680 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2681 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2682 */
2683#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2684 if (HMVMX_IS_64BIT_HOST_MODE())
2685 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2686 else
2687 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2688#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2689 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2690 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2691 else
2692 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2693#endif
2694
2695 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2696 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2697
2698 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2699 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2700 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2701 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2702 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2703
2704 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2705 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2706
2707 if ((val & zap) != val)
2708 {
2709 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2710 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2711 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2712 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2713 }
2714
2715 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2716 AssertRCReturn(rc, rc);
2717
2718 /* Update VCPU with the currently set VM-exit controls. */
2719 pVCpu->hm.s.vmx.u32ExitCtls = val;
2720 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2721 }
2722 return rc;
2723}
2724
2725
2726/**
2727 * Loads the guest APIC and related state.
2728 *
2729 * @returns VBox status code.
2730 * @param pVM Pointer to the VM.
2731 * @param pVCpu Pointer to the VMCPU.
2732 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2733 * out-of-sync. Make sure to update the required fields
2734 * before using them.
2735 */
2736DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2737{
2738 int rc = VINF_SUCCESS;
2739 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2740 {
2741 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2742 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2743 {
2744 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2745
2746 bool fPendingIntr = false;
2747 uint8_t u8Tpr = 0;
2748 uint8_t u8PendingIntr = 0;
2749 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2750 AssertRCReturn(rc, rc);
2751
2752 /*
2753 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2754 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2755 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2756 * the interrupt when we VM-exit for other reasons.
2757 */
2758 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2759 uint32_t u32TprThreshold = 0;
2760 if (fPendingIntr)
2761 {
2762 /* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2763 const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2764 const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2765 if (u8PendingPriority <= u8TprPriority)
2766 u32TprThreshold = u8PendingPriority;
2767 else
2768 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2769 }
2770 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2771
2772 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2773 AssertRCReturn(rc, rc);
2774 }
2775
2776 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2777 }
2778 return rc;
2779}
2780
2781
2782/**
2783 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2784 *
2785 * @returns Guest's interruptibility-state.
2786 * @param pVCpu Pointer to the VMCPU.
2787 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2788 * out-of-sync. Make sure to update the required fields
2789 * before using them.
2790 *
2791 * @remarks No-long-jump zone!!!
2792 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2793 */
2794DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2795{
2796 /*
2797 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2798 * inhibit interrupts or clear any existing interrupt-inhibition.
2799 */
2800 uint32_t uIntrState = 0;
2801 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2802 {
2803 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2804 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
2805 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2806 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2807 {
2808 /*
2809 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2810 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2811 */
2812 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2813 }
2814 else if (pMixedCtx->eflags.Bits.u1IF)
2815 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2816 else
2817 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2818 }
2819 return uIntrState;
2820}
2821
2822
2823/**
2824 * Loads the guest's interruptibility-state into the guest-state area in the
2825 * VMCS.
2826 *
2827 * @returns VBox status code.
2828 * @param pVCpu Pointer to the VMCPU.
2829 * @param uIntrState The interruptibility-state to set.
2830 */
2831static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2832{
2833 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2834 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2835 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2836 AssertRCReturn(rc, rc);
2837 return rc;
2838}
2839
2840
2841/**
2842 * Loads the guest's RIP into the guest-state area in the VMCS.
2843 *
2844 * @returns VBox status code.
2845 * @param pVCpu Pointer to the VMCPU.
2846 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2847 * out-of-sync. Make sure to update the required fields
2848 * before using them.
2849 *
2850 * @remarks No-long-jump zone!!!
2851 */
2852static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2853{
2854 int rc = VINF_SUCCESS;
2855 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2856 {
2857 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2858 AssertRCReturn(rc, rc);
2859 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2860 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#x\n", pMixedCtx->rip, pVCpu->hm.s.fContextUseFlags));
2861 }
2862 return rc;
2863}
2864
2865
2866/**
2867 * Loads the guest's RSP into the guest-state area in the VMCS.
2868 *
2869 * @returns VBox status code.
2870 * @param pVCpu Pointer to the VMCPU.
2871 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2872 * out-of-sync. Make sure to update the required fields
2873 * before using them.
2874 *
2875 * @remarks No-long-jump zone!!!
2876 */
2877static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2878{
2879 int rc = VINF_SUCCESS;
2880 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2881 {
2882 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2883 AssertRCReturn(rc, rc);
2884 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2885 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2886 }
2887 return rc;
2888}
2889
2890
2891/**
2892 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
2893 *
2894 * @returns VBox status code.
2895 * @param pVCpu Pointer to the VMCPU.
2896 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2897 * out-of-sync. Make sure to update the required fields
2898 * before using them.
2899 *
2900 * @remarks No-long-jump zone!!!
2901 */
2902static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2903{
2904 int rc = VINF_SUCCESS;
2905 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2906 {
2907 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2908 Let us assert it as such and use 32-bit VMWRITE. */
2909 Assert(!(pMixedCtx->rflags.u64 >> 32));
2910 X86EFLAGS Eflags = pMixedCtx->eflags;
2911 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2912 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2913
2914 /*
2915 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2916 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2917 */
2918 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2919 {
2920 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2921 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2922 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2923 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2924 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2925 }
2926
2927 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2928 AssertRCReturn(rc, rc);
2929
2930 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2931 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2932 }
2933 return rc;
2934}
2935
2936
2937/**
2938 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2939 *
2940 * @returns VBox status code.
2941 * @param pVCpu Pointer to the VMCPU.
2942 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2943 * out-of-sync. Make sure to update the required fields
2944 * before using them.
2945 *
2946 * @remarks No-long-jump zone!!!
2947 */
2948DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2949{
2950 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2951 AssertRCReturn(rc, rc);
2952 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2953 AssertRCReturn(rc, rc);
2954 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2955 AssertRCReturn(rc, rc);
2956 return rc;
2957}
2958
2959
2960/**
2961 * Loads the guest CR0 control register into the guest-state area in the VMCS.
2962 * CR0 is partially shared with the host and we have to consider the FPU bits.
2963 *
2964 * @returns VBox status code.
2965 * @param pVM Pointer to the VM.
2966 * @param pVCpu Pointer to the VMCPU.
2967 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2968 * out-of-sync. Make sure to update the required fields
2969 * before using them.
2970 *
2971 * @remarks No-long-jump zone!!!
2972 */
2973static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2974{
2975 /*
2976 * Guest CR0.
2977 * Guest FPU.
2978 */
2979 int rc = VINF_SUCCESS;
2980 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2981 {
2982 Assert(!(pMixedCtx->cr0 >> 32));
2983 uint32_t u32GuestCR0 = pMixedCtx->cr0;
2984 PVM pVM = pVCpu->CTX_SUFF(pVM);
2985
2986 /* The guest's view (read access) of its CR0 is unblemished. */
2987 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2988 AssertRCReturn(rc, rc);
2989 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2990
2991 /* Setup VT-x's view of the guest CR0. */
2992 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2993 if (pVM->hm.s.fNestedPaging)
2994 {
2995 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
2996 {
2997 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2998 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2999 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3000 }
3001 else
3002 {
3003 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3004 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3005 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3006 }
3007
3008 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3009 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3010 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3011
3012 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3013 AssertRCReturn(rc, rc);
3014 }
3015 else
3016 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3017
3018 /*
3019 * Guest FPU bits.
3020 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3021 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3022 */
3023 u32GuestCR0 |= X86_CR0_NE;
3024 bool fInterceptNM = false;
3025 if (CPUMIsGuestFPUStateActive(pVCpu))
3026 {
3027 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3028 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3029 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3030 }
3031 else
3032 {
3033 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3034 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3035 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3036 }
3037
3038 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3039 bool fInterceptMF = false;
3040 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3041 fInterceptMF = true;
3042
3043 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3044 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3045 {
3046 Assert(PDMVmmDevHeapIsEnabled(pVM));
3047 Assert(pVM->hm.s.vmx.pRealModeTSS);
3048 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3049 fInterceptNM = true;
3050 fInterceptMF = true;
3051 }
3052 else
3053 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3054
3055 if (fInterceptNM)
3056 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3057 else
3058 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3059
3060 if (fInterceptMF)
3061 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3062 else
3063 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3064
3065 /* Additional intercepts for debugging, define these yourself explicitly. */
3066#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3067 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3068 | RT_BIT(X86_XCPT_BP)
3069 | RT_BIT(X86_XCPT_DB)
3070 | RT_BIT(X86_XCPT_DE)
3071 | RT_BIT(X86_XCPT_NM)
3072 | RT_BIT(X86_XCPT_UD)
3073 | RT_BIT(X86_XCPT_NP)
3074 | RT_BIT(X86_XCPT_SS)
3075 | RT_BIT(X86_XCPT_GP)
3076 | RT_BIT(X86_XCPT_PF)
3077 | RT_BIT(X86_XCPT_MF)
3078 ;
3079#elif defined(HMVMX_ALWAYS_TRAP_PF)
3080 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3081#endif
3082
3083 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3084
3085 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3086 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3087 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3088 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3089 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3090 else
3091 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3092
3093 u32GuestCR0 |= uSetCR0;
3094 u32GuestCR0 &= uZapCR0;
3095 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3096
3097 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3098 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3099 AssertRCReturn(rc, rc);
3100 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3101 AssertRCReturn(rc, rc);
3102 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3103
3104 /*
3105 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3106 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3107 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3108 */
3109 uint32_t u32CR0Mask = 0;
3110 u32CR0Mask = X86_CR0_PE
3111 | X86_CR0_NE
3112 | X86_CR0_WP
3113 | X86_CR0_PG
3114 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3115 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3116 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3117 /** @todo Temporarily intercept CR0.PE changes with unrestricted. Fix PGM
3118 * enmGuestMode to not be out-of-sync. See @bugref{6398}. */
3119#if 0
3120 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3121 u32CR0Mask &= ~X86_CR0_PE;
3122#endif
3123 if (pVM->hm.s.fNestedPaging)
3124 u32CR0Mask &= ~X86_CR0_WP;
3125
3126 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3127 if (fInterceptNM)
3128 u32CR0Mask |= (X86_CR0_TS | X86_CR0_MP);
3129 else
3130 u32CR0Mask &= ~(X86_CR0_TS | X86_CR0_MP);
3131
3132 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3133 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3134 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3135 AssertRCReturn(rc, rc);
3136
3137 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3138 }
3139 return rc;
3140}
3141
3142
3143/**
3144 * Loads the guest control registers (CR3, CR4) into the guest-state area
3145 * in the VMCS.
3146 *
3147 * @returns VBox status code.
3148 * @param pVM Pointer to the VM.
3149 * @param pVCpu Pointer to the VMCPU.
3150 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3151 * out-of-sync. Make sure to update the required fields
3152 * before using them.
3153 *
3154 * @remarks No-long-jump zone!!!
3155 */
3156static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3157{
3158 int rc = VINF_SUCCESS;
3159 PVM pVM = pVCpu->CTX_SUFF(pVM);
3160
3161 /*
3162 * Guest CR2.
3163 * It's always loaded in the assembler code. Nothing to do here.
3164 */
3165
3166 /*
3167 * Guest CR3.
3168 */
3169 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3170 {
3171 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3172 if (pVM->hm.s.fNestedPaging)
3173 {
3174 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3175
3176 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3177 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3178 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3179 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3180
3181 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3182 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3183 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3184
3185 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3186 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3187 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3188 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3189
3190 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3191 AssertRCReturn(rc, rc);
3192 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3193
3194 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3195 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3196 {
3197 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3198 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3199 {
3200 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3201 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3202 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3203 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3204 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3205 }
3206
3207 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3208 have Unrestricted Execution to handle the guest when it's not using paging. */
3209 GCPhysGuestCR3 = pMixedCtx->cr3;
3210 }
3211 else
3212 {
3213 /*
3214 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3215 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3216 * EPT takes care of translating it to host-physical addresses.
3217 */
3218 RTGCPHYS GCPhys;
3219 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3220 Assert(PDMVmmDevHeapIsEnabled(pVM));
3221
3222 /* We obtain it here every time as the guest could have relocated this PCI region. */
3223 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3224 AssertRCReturn(rc, rc);
3225
3226 GCPhysGuestCR3 = GCPhys;
3227 }
3228
3229 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3230 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3231 }
3232 else
3233 {
3234 /* Non-nested paging case, just use the hypervisor's CR3. */
3235 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3236
3237 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3238 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3239 }
3240 AssertRCReturn(rc, rc);
3241
3242 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3243 }
3244
3245 /*
3246 * Guest CR4.
3247 */
3248 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3249 {
3250 Assert(!(pMixedCtx->cr4 >> 32));
3251 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3252
3253 /* The guest's view of its CR4 is unblemished. */
3254 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3255 AssertRCReturn(rc, rc);
3256 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3257
3258 /* Setup VT-x's view of the guest CR4. */
3259 /*
3260 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3261 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3262 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3263 */
3264 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3265 {
3266 Assert(pVM->hm.s.vmx.pRealModeTSS);
3267 Assert(PDMVmmDevHeapIsEnabled(pVM));
3268 u32GuestCR4 &= ~X86_CR4_VME;
3269 }
3270
3271 if (pVM->hm.s.fNestedPaging)
3272 {
3273 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3274 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3275 {
3276 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3277 u32GuestCR4 |= X86_CR4_PSE;
3278 /* Our identity mapping is a 32 bits page directory. */
3279 u32GuestCR4 &= ~X86_CR4_PAE;
3280 }
3281 /* else use guest CR4.*/
3282 }
3283 else
3284 {
3285 /*
3286 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3287 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3288 */
3289 switch (pVCpu->hm.s.enmShadowMode)
3290 {
3291 case PGMMODE_REAL: /* Real-mode. */
3292 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3293 case PGMMODE_32_BIT: /* 32-bit paging. */
3294 {
3295 u32GuestCR4 &= ~X86_CR4_PAE;
3296 break;
3297 }
3298
3299 case PGMMODE_PAE: /* PAE paging. */
3300 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3301 {
3302 u32GuestCR4 |= X86_CR4_PAE;
3303 break;
3304 }
3305
3306 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3307 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3308#ifdef VBOX_ENABLE_64_BITS_GUESTS
3309 break;
3310#endif
3311 default:
3312 AssertFailed();
3313 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3314 }
3315 }
3316
3317 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3318 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3319 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3320 u32GuestCR4 |= uSetCR4;
3321 u32GuestCR4 &= uZapCR4;
3322
3323 /* Write VT-x's view of the guest CR4 into the VMCS. */
3324 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3325 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3326 AssertRCReturn(rc, rc);
3327
3328 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3329 uint32_t u32CR4Mask = 0;
3330 u32CR4Mask = X86_CR4_VME
3331 | X86_CR4_PAE
3332 | X86_CR4_PGE
3333 | X86_CR4_PSE
3334 | X86_CR4_VMXE;
3335 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3336 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3337 AssertRCReturn(rc, rc);
3338
3339 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3340 }
3341 return rc;
3342}
3343
3344
3345/**
3346 * Loads the guest debug registers into the guest-state area in the VMCS.
3347 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3348 *
3349 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3350 *
3351 * @returns VBox status code.
3352 * @param pVCpu Pointer to the VMCPU.
3353 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3354 * out-of-sync. Make sure to update the required fields
3355 * before using them.
3356 *
3357 * @remarks No-long-jump zone!!!
3358 */
3359static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3360{
3361 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3362 return VINF_SUCCESS;
3363
3364#ifdef VBOX_STRICT
3365 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3366 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3367 {
3368 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3369 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3370 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3371 }
3372#endif
3373
3374 int rc;
3375 PVM pVM = pVCpu->CTX_SUFF(pVM);
3376 bool fInterceptDB = false;
3377 bool fInterceptMovDRx = false;
3378 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3379 {
3380 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3381 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3382 {
3383 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3384 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3385 AssertRCReturn(rc, rc);
3386 Assert(fInterceptDB == false);
3387 }
3388 else
3389 {
3390 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3391 pVCpu->hm.s.fClearTrapFlag = true;
3392 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RFLAGS;
3393 fInterceptDB = true;
3394 }
3395 }
3396
3397 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3398 {
3399 /*
3400 * Use the combined guest and host DRx values found in the hypervisor
3401 * register set because the debugger has breakpoints active or someone
3402 * is single stepping on the host side without a monitor trap flag.
3403 *
3404 * Note! DBGF expects a clean DR6 state before executing guest code.
3405 */
3406 if (!CPUMIsHyperDebugStateActive(pVCpu))
3407 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3408 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3409 Assert(CPUMIsHyperDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3410
3411 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3412 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3413 AssertRCReturn(rc, rc);
3414
3415 fInterceptDB = true;
3416 fInterceptMovDRx = true;
3417 }
3418 else
3419 {
3420 /*
3421 * If the guest has enabled debug registers, we need to load them prior to
3422 * executing guest code so they'll trigger at the right time.
3423 */
3424 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3425 {
3426 if (!CPUMIsGuestDebugStateActive(pVCpu))
3427 {
3428 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3429 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3430 }
3431 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3432 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3433 }
3434 /*
3435 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3436 * must intercept #DB in order to maintain a correct DR6 guest value.
3437 */
3438 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3439 {
3440 fInterceptMovDRx = true;
3441 fInterceptDB = true;
3442 }
3443
3444 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3445 AssertRCReturn(rc, rc);
3446 }
3447
3448 /*
3449 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3450 */
3451 if (fInterceptDB)
3452 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3453 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3454 {
3455#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3456 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3457#endif
3458 }
3459 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3460 AssertRCReturn(rc, rc);
3461
3462 /*
3463 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3464 */
3465 if (fInterceptMovDRx)
3466 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3467 else
3468 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3469 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3470 AssertRCReturn(rc, rc);
3471
3472 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3473 return VINF_SUCCESS;
3474}
3475
3476
3477#ifdef VBOX_STRICT
3478/**
3479 * Strict function to validate segment registers.
3480 *
3481 * @remarks ASSUMES CR0 is up to date.
3482 */
3483static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3484{
3485 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3486 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3487 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3488 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3489 && ( !CPUMIsGuestInRealModeEx(pCtx)
3490 && !CPUMIsGuestInV86ModeEx(pCtx)))
3491 {
3492 /* Protected mode checks */
3493 /* CS */
3494 Assert(pCtx->cs.Attr.n.u1Present);
3495 Assert(!(pCtx->cs.Attr.u & 0xf00));
3496 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3497 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3498 || !(pCtx->cs.Attr.n.u1Granularity));
3499 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3500 || (pCtx->cs.Attr.n.u1Granularity));
3501 /* CS cannot be loaded with NULL in protected mode. */
3502 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3503 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3504 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3505 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3506 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3507 else
3508 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3509 /* SS */
3510 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3511 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3512 if ( !(pCtx->cr0 & X86_CR0_PE)
3513 || pCtx->cs.Attr.n.u4Type == 3)
3514 {
3515 Assert(!pCtx->ss.Attr.n.u2Dpl);
3516 }
3517 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3518 {
3519 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3520 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3521 Assert(pCtx->ss.Attr.n.u1Present);
3522 Assert(!(pCtx->ss.Attr.u & 0xf00));
3523 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3524 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3525 || !(pCtx->ss.Attr.n.u1Granularity));
3526 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3527 || (pCtx->ss.Attr.n.u1Granularity));
3528 }
3529 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3530 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3531 {
3532 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3533 Assert(pCtx->ds.Attr.n.u1Present);
3534 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3535 Assert(!(pCtx->ds.Attr.u & 0xf00));
3536 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3537 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3538 || !(pCtx->ds.Attr.n.u1Granularity));
3539 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3540 || (pCtx->ds.Attr.n.u1Granularity));
3541 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3542 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3543 }
3544 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3545 {
3546 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3547 Assert(pCtx->es.Attr.n.u1Present);
3548 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3549 Assert(!(pCtx->es.Attr.u & 0xf00));
3550 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3551 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3552 || !(pCtx->es.Attr.n.u1Granularity));
3553 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3554 || (pCtx->es.Attr.n.u1Granularity));
3555 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3556 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3557 }
3558 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3559 {
3560 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3561 Assert(pCtx->fs.Attr.n.u1Present);
3562 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3563 Assert(!(pCtx->fs.Attr.u & 0xf00));
3564 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3565 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3566 || !(pCtx->fs.Attr.n.u1Granularity));
3567 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3568 || (pCtx->fs.Attr.n.u1Granularity));
3569 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3570 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3571 }
3572 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3573 {
3574 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3575 Assert(pCtx->gs.Attr.n.u1Present);
3576 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3577 Assert(!(pCtx->gs.Attr.u & 0xf00));
3578 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3579 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3580 || !(pCtx->gs.Attr.n.u1Granularity));
3581 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3582 || (pCtx->gs.Attr.n.u1Granularity));
3583 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3584 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3585 }
3586 /* 64-bit capable CPUs. */
3587# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3588 Assert(!(pCtx->cs.u64Base >> 32));
3589 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3590 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3591 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3592# endif
3593 }
3594 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3595 || ( CPUMIsGuestInRealModeEx(pCtx)
3596 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3597 {
3598 /* Real and v86 mode checks. */
3599 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3600 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3601 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3602 {
3603 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3604 }
3605 else
3606 {
3607 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3608 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3609 }
3610
3611 /* CS */
3612 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3613 Assert(pCtx->cs.u32Limit == 0xffff);
3614 Assert(u32CSAttr == 0xf3);
3615 /* SS */
3616 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3617 Assert(pCtx->ss.u32Limit == 0xffff);
3618 Assert(u32SSAttr == 0xf3);
3619 /* DS */
3620 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3621 Assert(pCtx->ds.u32Limit == 0xffff);
3622 Assert(u32DSAttr == 0xf3);
3623 /* ES */
3624 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3625 Assert(pCtx->es.u32Limit == 0xffff);
3626 Assert(u32ESAttr == 0xf3);
3627 /* FS */
3628 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3629 Assert(pCtx->fs.u32Limit == 0xffff);
3630 Assert(u32FSAttr == 0xf3);
3631 /* GS */
3632 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3633 Assert(pCtx->gs.u32Limit == 0xffff);
3634 Assert(u32GSAttr == 0xf3);
3635 /* 64-bit capable CPUs. */
3636# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3637 Assert(!(pCtx->cs.u64Base >> 32));
3638 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3639 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3640 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3641# endif
3642 }
3643}
3644#endif /* VBOX_STRICT */
3645
3646
3647/**
3648 * Writes a guest segment register into the guest-state area in the VMCS.
3649 *
3650 * @returns VBox status code.
3651 * @param pVCpu Pointer to the VMCPU.
3652 * @param idxSel Index of the selector in the VMCS.
3653 * @param idxLimit Index of the segment limit in the VMCS.
3654 * @param idxBase Index of the segment base in the VMCS.
3655 * @param idxAccess Index of the access rights of the segment in the VMCS.
3656 * @param pSelReg Pointer to the segment selector.
3657 * @param pCtx Pointer to the guest-CPU context.
3658 *
3659 * @remarks No-long-jump zone!!!
3660 */
3661static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3662 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3663{
3664 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3665 AssertRCReturn(rc, rc);
3666 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3667 AssertRCReturn(rc, rc);
3668 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3669 AssertRCReturn(rc, rc);
3670
3671 uint32_t u32Access = pSelReg->Attr.u;
3672 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3673 {
3674 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3675 u32Access = 0xf3;
3676 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3677 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3678 }
3679 else
3680 {
3681 /*
3682 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3683 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3684 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3685 * loaded in protected-mode have their attribute as 0.
3686 */
3687 if (!u32Access)
3688 u32Access = X86DESCATTR_UNUSABLE;
3689 }
3690
3691 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3692 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3693 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3694
3695 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3696 AssertRCReturn(rc, rc);
3697 return rc;
3698}
3699
3700
3701/**
3702 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3703 * into the guest-state area in the VMCS.
3704 *
3705 * @returns VBox status code.
3706 * @param pVM Pointer to the VM.
3707 * @param pVCPU Pointer to the VMCPU.
3708 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3709 * out-of-sync. Make sure to update the required fields
3710 * before using them.
3711 *
3712 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
3713 * @remarks No-long-jump zone!!!
3714 */
3715static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3716{
3717 int rc = VERR_INTERNAL_ERROR_5;
3718 PVM pVM = pVCpu->CTX_SUFF(pVM);
3719
3720 /*
3721 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
3722 */
3723 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3724 {
3725 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3726 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3727 {
3728 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3729 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3730 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3731 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3732 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3733 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3734 }
3735
3736#ifdef VBOX_WITH_REM
3737 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3738 {
3739 Assert(pVM->hm.s.vmx.pRealModeTSS);
3740 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3741 if ( pVCpu->hm.s.vmx.fWasInRealMode
3742 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3743 {
3744 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3745 in real-mode (e.g. OpenBSD 4.0) */
3746 REMFlushTBs(pVM);
3747 Log4(("Load: Switch to protected mode detected!\n"));
3748 pVCpu->hm.s.vmx.fWasInRealMode = false;
3749 }
3750 }
3751#endif
3752 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3753 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3754 AssertRCReturn(rc, rc);
3755 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3756 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3757 AssertRCReturn(rc, rc);
3758 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3759 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3760 AssertRCReturn(rc, rc);
3761 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3762 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3763 AssertRCReturn(rc, rc);
3764 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3765 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3766 AssertRCReturn(rc, rc);
3767 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3768 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3769 AssertRCReturn(rc, rc);
3770
3771 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3772 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3773#ifdef VBOX_STRICT
3774 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3775#endif
3776 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3777 }
3778
3779 /*
3780 * Guest TR.
3781 */
3782 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3783 {
3784 /*
3785 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3786 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3787 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3788 */
3789 uint16_t u16Sel = 0;
3790 uint32_t u32Limit = 0;
3791 uint64_t u64Base = 0;
3792 uint32_t u32AccessRights = 0;
3793
3794 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3795 {
3796 u16Sel = pMixedCtx->tr.Sel;
3797 u32Limit = pMixedCtx->tr.u32Limit;
3798 u64Base = pMixedCtx->tr.u64Base;
3799 u32AccessRights = pMixedCtx->tr.Attr.u;
3800 }
3801 else
3802 {
3803 Assert(pVM->hm.s.vmx.pRealModeTSS);
3804 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3805
3806 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3807 RTGCPHYS GCPhys;
3808 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3809 AssertRCReturn(rc, rc);
3810
3811 X86DESCATTR DescAttr;
3812 DescAttr.u = 0;
3813 DescAttr.n.u1Present = 1;
3814 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3815
3816 u16Sel = 0;
3817 u32Limit = HM_VTX_TSS_SIZE;
3818 u64Base = GCPhys; /* in real-mode phys = virt. */
3819 u32AccessRights = DescAttr.u;
3820 }
3821
3822 /* Validate. */
3823 Assert(!(u16Sel & RT_BIT(2)));
3824 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3825 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3826 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3827 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3828 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3829 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3830 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3831 Assert( (u32Limit & 0xfff) == 0xfff
3832 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3833 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3834 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3835
3836 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3837 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3838 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3839 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3840
3841 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3842 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3843 }
3844
3845 /*
3846 * Guest GDTR.
3847 */
3848 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3849 {
3850 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3851 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3852
3853 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3854 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3855 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3856 }
3857
3858 /*
3859 * Guest LDTR.
3860 */
3861 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3862 {
3863 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3864 uint32_t u32Access = 0;
3865 if (!pMixedCtx->ldtr.Attr.u)
3866 u32Access = X86DESCATTR_UNUSABLE;
3867 else
3868 u32Access = pMixedCtx->ldtr.Attr.u;
3869
3870 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3871 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3872 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3873 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3874
3875 /* Validate. */
3876 if (!(u32Access & X86DESCATTR_UNUSABLE))
3877 {
3878 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3879 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3880 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3881 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3882 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3883 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3884 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3885 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3886 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3887 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3888 }
3889
3890 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3891 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3892 }
3893
3894 /*
3895 * Guest IDTR.
3896 */
3897 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3898 {
3899 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3900 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3901
3902 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3903 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3904 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3905 }
3906
3907 return VINF_SUCCESS;
3908}
3909
3910
3911/**
3912 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3913 * areas. These MSRs will automatically be loaded to the host CPU on every
3914 * successful VM entry and stored from the host CPU on every successful VM exit.
3915 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
3916 *
3917 * @returns VBox status code.
3918 * @param pVCpu Pointer to the VMCPU.
3919 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3920 * out-of-sync. Make sure to update the required fields
3921 * before using them.
3922 *
3923 * @remarks No-long-jump zone!!!
3924 */
3925static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3926{
3927 AssertPtr(pVCpu);
3928 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3929
3930 /*
3931 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3932 */
3933 int rc = VINF_SUCCESS;
3934 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3935 {
3936#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3937 PVM pVM = pVCpu->CTX_SUFF(pVM);
3938 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3939 uint32_t cGuestMsrs = 0;
3940
3941 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3942 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3943 * when the guest really is in 64-bit mode. */
3944 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3945 if (fSupportsLongMode)
3946 {
3947 pGuestMsr->u32Msr = MSR_K8_LSTAR;
3948 pGuestMsr->u32Reserved = 0;
3949 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3950 pGuestMsr++; cGuestMsrs++;
3951 pGuestMsr->u32Msr = MSR_K6_STAR;
3952 pGuestMsr->u32Reserved = 0;
3953 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3954 pGuestMsr++; cGuestMsrs++;
3955 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
3956 pGuestMsr->u32Reserved = 0;
3957 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3958 pGuestMsr++; cGuestMsrs++;
3959 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
3960 pGuestMsr->u32Reserved = 0;
3961 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3962 pGuestMsr++; cGuestMsrs++;
3963 }
3964
3965 /*
3966 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3967 * load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3968 */
3969 if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3970 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3971 {
3972 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
3973 pGuestMsr->u32Reserved = 0;
3974 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3975 AssertRCReturn(rc, rc);
3976 pGuestMsr++; cGuestMsrs++;
3977 }
3978
3979 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3980 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
3981 {
3982 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3983 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
3984 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3985 }
3986
3987 /* Update the VCPU's copy of the guest MSR count. */
3988 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3989 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3990 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3991#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3992
3993 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3994 }
3995
3996 /*
3997 * Guest Sysenter MSRs.
3998 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3999 * VM-exits on WRMSRs for these MSRs.
4000 */
4001 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4002 {
4003 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4004 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
4005 }
4006 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4007 {
4008 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4009 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
4010 }
4011 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4012 {
4013 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4014 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
4015 }
4016
4017 return rc;
4018}
4019
4020
4021/**
4022 * Loads the guest activity state into the guest-state area in the VMCS.
4023 *
4024 * @returns VBox status code.
4025 * @param pVCpu Pointer to the VMCPU.
4026 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4027 * out-of-sync. Make sure to update the required fields
4028 * before using them.
4029 *
4030 * @remarks No-long-jump zone!!!
4031 */
4032static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4033{
4034 /** @todo See if we can make use of other states, e.g.
4035 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4036 int rc = VINF_SUCCESS;
4037 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
4038 {
4039 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4040 AssertRCReturn(rc, rc);
4041 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
4042 }
4043 return rc;
4044}
4045
4046
4047/**
4048 * Sets up the appropriate function to run guest code.
4049 *
4050 * @returns VBox status code.
4051 * @param pVCpu Pointer to the VMCPU.
4052 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4053 * out-of-sync. Make sure to update the required fields
4054 * before using them.
4055 *
4056 * @remarks No-long-jump zone!!!
4057 */
4058static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4059{
4060 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4061 {
4062#ifndef VBOX_ENABLE_64_BITS_GUESTS
4063 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4064#endif
4065 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4066#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4067 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4068 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4069 {
4070 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4071 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS;
4072 }
4073#else
4074 /* 64-bit host or hybrid host. */
4075 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4076#endif
4077 }
4078 else
4079 {
4080 /* Guest is not in long mode, use the 32-bit handler. */
4081#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4082 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4083 {
4084 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4085 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS;
4086 }
4087#else
4088 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4089#endif
4090 }
4091 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4092 return VINF_SUCCESS;
4093}
4094
4095
4096/**
4097 * Wrapper for running the guest code in VT-x.
4098 *
4099 * @returns VBox strict status code.
4100 * @param pVM Pointer to the VM.
4101 * @param pVCpu Pointer to the VMCPU.
4102 * @param pCtx Pointer to the guest-CPU context.
4103 *
4104 * @remarks No-long-jump zone!!!
4105 */
4106DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4107{
4108 /*
4109 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4110 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4111 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4112 */
4113 const bool fResumeVM = !!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4114 /** @todo Add stats for resume vs launch. */
4115#ifdef VBOX_WITH_KERNEL_USING_XMM
4116 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4117#else
4118 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4119#endif
4120}
4121
4122
4123/**
4124 * Reports world-switch error and dumps some useful debug info.
4125 *
4126 * @param pVM Pointer to the VM.
4127 * @param pVCpu Pointer to the VMCPU.
4128 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4129 * @param pCtx Pointer to the guest-CPU context.
4130 * @param pVmxTransient Pointer to the VMX transient structure (only
4131 * exitReason updated).
4132 */
4133static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4134{
4135 Assert(pVM);
4136 Assert(pVCpu);
4137 Assert(pCtx);
4138 Assert(pVmxTransient);
4139 HMVMX_ASSERT_PREEMPT_SAFE();
4140
4141 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4142 switch (rcVMRun)
4143 {
4144 case VERR_VMX_INVALID_VMXON_PTR:
4145 AssertFailed();
4146 break;
4147 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4148 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4149 {
4150 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4151 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4152 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4153 AssertRC(rc);
4154
4155 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4156 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4157 Cannot do it here as we may have been long preempted. */
4158
4159#ifdef VBOX_STRICT
4160 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4161 pVmxTransient->uExitReason));
4162 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4163 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4164 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4165 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4166 else
4167 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4168 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4169 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4170
4171 /* VMX control bits. */
4172 uint32_t u32Val;
4173 uint64_t u64Val;
4174 HMVMXHCUINTREG uHCReg;
4175 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4176 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4177 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4178 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4179 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4180 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4181 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4182 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4183 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4184 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4185 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4186 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4187 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4188 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4189 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4190 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4191 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4192 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4193 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4194 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4195 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4196 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4197 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4198 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4199 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4200 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4201 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4202 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4203 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4204 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4205 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4206 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4207 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4208 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4209 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4210 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4211 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4212 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4213 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4214 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4215 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4216 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4217
4218 /* Guest bits. */
4219 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4220 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4221 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4222 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4223 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4224 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4225 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4226 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4227
4228 /* Host bits. */
4229 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4230 Log4(("Host CR0 %#RHr\n", uHCReg));
4231 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4232 Log4(("Host CR3 %#RHr\n", uHCReg));
4233 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4234 Log4(("Host CR4 %#RHr\n", uHCReg));
4235
4236 RTGDTR HostGdtr;
4237 PCX86DESCHC pDesc;
4238 ASMGetGDTR(&HostGdtr);
4239 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4240 Log4(("Host CS %#08x\n", u32Val));
4241 if (u32Val < HostGdtr.cbGdt)
4242 {
4243 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4244 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4245 }
4246
4247 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4248 Log4(("Host DS %#08x\n", u32Val));
4249 if (u32Val < HostGdtr.cbGdt)
4250 {
4251 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4252 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4253 }
4254
4255 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4256 Log4(("Host ES %#08x\n", u32Val));
4257 if (u32Val < HostGdtr.cbGdt)
4258 {
4259 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4260 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4261 }
4262
4263 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4264 Log4(("Host FS %#08x\n", u32Val));
4265 if (u32Val < HostGdtr.cbGdt)
4266 {
4267 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4268 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4269 }
4270
4271 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4272 Log4(("Host GS %#08x\n", u32Val));
4273 if (u32Val < HostGdtr.cbGdt)
4274 {
4275 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4276 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4277 }
4278
4279 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4280 Log4(("Host SS %#08x\n", u32Val));
4281 if (u32Val < HostGdtr.cbGdt)
4282 {
4283 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4284 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4285 }
4286
4287 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4288 Log4(("Host TR %#08x\n", u32Val));
4289 if (u32Val < HostGdtr.cbGdt)
4290 {
4291 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4292 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4293 }
4294
4295 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4296 Log4(("Host TR Base %#RHv\n", uHCReg));
4297 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4298 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4299 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4300 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4301 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4302 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4303 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4304 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4305 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4306 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4307 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4308 Log4(("Host RSP %#RHv\n", uHCReg));
4309 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4310 Log4(("Host RIP %#RHv\n", uHCReg));
4311# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4312 if (HMVMX_IS_64BIT_HOST_MODE())
4313 {
4314 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4315 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4316 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4317 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4318 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4319 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4320 }
4321# endif
4322#endif /* VBOX_STRICT */
4323 break;
4324 }
4325
4326 default:
4327 /* Impossible */
4328 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4329 break;
4330 }
4331 NOREF(pVM);
4332}
4333
4334
4335#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4336#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4337# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4338#endif
4339#ifdef VBOX_STRICT
4340static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4341{
4342 switch (idxField)
4343 {
4344 case VMX_VMCS_GUEST_RIP:
4345 case VMX_VMCS_GUEST_RSP:
4346 case VMX_VMCS_GUEST_SYSENTER_EIP:
4347 case VMX_VMCS_GUEST_SYSENTER_ESP:
4348 case VMX_VMCS_GUEST_GDTR_BASE:
4349 case VMX_VMCS_GUEST_IDTR_BASE:
4350 case VMX_VMCS_GUEST_CS_BASE:
4351 case VMX_VMCS_GUEST_DS_BASE:
4352 case VMX_VMCS_GUEST_ES_BASE:
4353 case VMX_VMCS_GUEST_FS_BASE:
4354 case VMX_VMCS_GUEST_GS_BASE:
4355 case VMX_VMCS_GUEST_SS_BASE:
4356 case VMX_VMCS_GUEST_LDTR_BASE:
4357 case VMX_VMCS_GUEST_TR_BASE:
4358 case VMX_VMCS_GUEST_CR3:
4359 return true;
4360 }
4361 return false;
4362}
4363
4364static bool hmR0VmxIsValidReadField(uint32_t idxField)
4365{
4366 switch (idxField)
4367 {
4368 /* Read-only fields. */
4369 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4370 return true;
4371 }
4372 /* Remaining readable fields should also be writable. */
4373 return hmR0VmxIsValidWriteField(idxField);
4374}
4375#endif /* VBOX_STRICT */
4376
4377
4378/**
4379 * Executes the specified handler in 64-bit mode.
4380 *
4381 * @returns VBox status code.
4382 * @param pVM Pointer to the VM.
4383 * @param pVCpu Pointer to the VMCPU.
4384 * @param pCtx Pointer to the guest CPU context.
4385 * @param enmOp The operation to perform.
4386 * @param cbParam Number of parameters.
4387 * @param paParam Array of 32-bit parameters.
4388 */
4389VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4390 uint32_t *paParam)
4391{
4392 int rc, rc2;
4393 PHMGLOBALCPUINFO pCpu;
4394 RTHCPHYS HCPhysCpuPage;
4395 RTCCUINTREG uOldEflags;
4396
4397 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4398 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4399 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4400 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4401
4402#ifdef VBOX_STRICT
4403 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4404 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4405
4406 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4407 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4408#endif
4409
4410 /* Disable interrupts. */
4411 uOldEflags = ASMIntDisableFlags();
4412
4413#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4414 RTCPUID idHostCpu = RTMpCpuId();
4415 CPUMR0SetLApic(pVCpu, idHostCpu);
4416#endif
4417
4418 pCpu = HMR0GetCurrentCpu();
4419 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4420
4421 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4422 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4423
4424 /* Leave VMX Root Mode. */
4425 VMXDisable();
4426
4427 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4428
4429 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4430 CPUMSetHyperEIP(pVCpu, enmOp);
4431 for (int i = (int)cbParam - 1; i >= 0; i--)
4432 CPUMPushHyper(pVCpu, paParam[i]);
4433
4434 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4435
4436 /* Call the switcher. */
4437 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4438 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4439
4440 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4441 /* Make sure the VMX instructions don't cause #UD faults. */
4442 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4443
4444 /* Re-enter VMX Root Mode */
4445 rc2 = VMXEnable(HCPhysCpuPage);
4446 if (RT_FAILURE(rc2))
4447 {
4448 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4449 ASMSetFlags(uOldEflags);
4450 return rc2;
4451 }
4452
4453 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4454 AssertRC(rc2);
4455 Assert(!(ASMGetFlags() & X86_EFL_IF));
4456 ASMSetFlags(uOldEflags);
4457 return rc;
4458}
4459
4460
4461/**
4462 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4463 * supporting 64-bit guests.
4464 *
4465 * @returns VBox status code.
4466 * @param fResume Whether to VMLAUNCH or VMRESUME.
4467 * @param pCtx Pointer to the guest-CPU context.
4468 * @param pCache Pointer to the VMCS cache.
4469 * @param pVM Pointer to the VM.
4470 * @param pVCpu Pointer to the VMCPU.
4471 */
4472DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4473{
4474 uint32_t aParam[6];
4475 PHMGLOBALCPUINFO pCpu = NULL;
4476 RTHCPHYS HCPhysCpuPage = 0;
4477 int rc = VERR_INTERNAL_ERROR_5;
4478
4479 pCpu = HMR0GetCurrentCpu();
4480 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4481
4482#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4483 pCache->uPos = 1;
4484 pCache->interPD = PGMGetInterPaeCR3(pVM);
4485 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4486#endif
4487
4488#ifdef VBOX_STRICT
4489 pCache->TestIn.HCPhysCpuPage = 0;
4490 pCache->TestIn.HCPhysVmcs = 0;
4491 pCache->TestIn.pCache = 0;
4492 pCache->TestOut.HCPhysVmcs = 0;
4493 pCache->TestOut.pCache = 0;
4494 pCache->TestOut.pCtx = 0;
4495 pCache->TestOut.eflags = 0;
4496#endif
4497
4498 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4499 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4500 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4501 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4502 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4503 aParam[5] = 0;
4504
4505#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4506 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4507 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4508#endif
4509 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4510
4511#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4512 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4513 Assert(pCtx->dr[4] == 10);
4514 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4515#endif
4516
4517#ifdef VBOX_STRICT
4518 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4519 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4520 pVCpu->hm.s.vmx.HCPhysVmcs));
4521 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4522 pCache->TestOut.HCPhysVmcs));
4523 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4524 pCache->TestOut.pCache));
4525 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4526 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4527 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4528 pCache->TestOut.pCtx));
4529 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4530#endif
4531 return rc;
4532}
4533
4534
4535/**
4536 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4537 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4538 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4539 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4540 *
4541 * @returns VBox status code.
4542 * @param pVM Pointer to the VM.
4543 * @param pVCpu Pointer to the VMCPU.
4544 */
4545static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4546{
4547#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4548{ \
4549 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4550 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4551 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4552 ++cReadFields; \
4553}
4554
4555 AssertPtr(pVM);
4556 AssertPtr(pVCpu);
4557 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4558 uint32_t cReadFields = 0;
4559
4560 /*
4561 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4562 * and serve to indicate exceptions to the rules.
4563 */
4564
4565 /* Guest-natural selector base fields. */
4566#if 0
4567 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4568 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4569 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4570#endif
4571 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4572 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4573 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4574 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4575 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4576 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4577 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4578 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4579 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4580 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4581 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4582 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4583#if 0
4584 /* Unused natural width guest-state fields. */
4585 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4586 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4587#endif
4588 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4589 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4590
4591 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4592#if 0
4593 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4594 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4595 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4596 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4597 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4598 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4599 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4600 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4601 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4602#endif
4603
4604 /* Natural width guest-state fields. */
4605 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4606#if 0
4607 /* Currently unused field. */
4608 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4609#endif
4610
4611 if (pVM->hm.s.fNestedPaging)
4612 {
4613 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4614 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4615 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4616 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4617 }
4618 else
4619 {
4620 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4621 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4622 }
4623
4624#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4625 return VINF_SUCCESS;
4626}
4627
4628
4629/**
4630 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4631 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4632 * darwin, running 64-bit guests).
4633 *
4634 * @returns VBox status code.
4635 * @param pVCpu Pointer to the VMCPU.
4636 * @param idxField The VMCS field encoding.
4637 * @param u64Val 16, 32 or 64 bits value.
4638 */
4639VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4640{
4641 int rc;
4642 switch (idxField)
4643 {
4644 /*
4645 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4646 */
4647 /* 64-bit Control fields. */
4648 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4649 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4650 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4651 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4652 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4653 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4654 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4655 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4656 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4657 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4658 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4659 case VMX_VMCS64_CTRL_EPTP_FULL:
4660 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4661 /* 64-bit Guest-state fields. */
4662 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4663 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4664 case VMX_VMCS64_GUEST_PAT_FULL:
4665 case VMX_VMCS64_GUEST_EFER_FULL:
4666 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4667 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4668 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4669 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4670 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4671 /* 64-bit Host-state fields. */
4672 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4673 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4674 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4675 {
4676 rc = VMXWriteVmcs32(idxField, u64Val);
4677 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4678 break;
4679 }
4680
4681 /*
4682 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4683 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4684 */
4685 /* Natural-width Guest-state fields. */
4686 case VMX_VMCS_GUEST_CR3:
4687 case VMX_VMCS_GUEST_ES_BASE:
4688 case VMX_VMCS_GUEST_CS_BASE:
4689 case VMX_VMCS_GUEST_SS_BASE:
4690 case VMX_VMCS_GUEST_DS_BASE:
4691 case VMX_VMCS_GUEST_FS_BASE:
4692 case VMX_VMCS_GUEST_GS_BASE:
4693 case VMX_VMCS_GUEST_LDTR_BASE:
4694 case VMX_VMCS_GUEST_TR_BASE:
4695 case VMX_VMCS_GUEST_GDTR_BASE:
4696 case VMX_VMCS_GUEST_IDTR_BASE:
4697 case VMX_VMCS_GUEST_RSP:
4698 case VMX_VMCS_GUEST_RIP:
4699 case VMX_VMCS_GUEST_SYSENTER_ESP:
4700 case VMX_VMCS_GUEST_SYSENTER_EIP:
4701 {
4702 if (!(u64Val >> 32))
4703 {
4704 /* If this field is 64-bit, VT-x will zero out the top bits. */
4705 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4706 }
4707 else
4708 {
4709 /* Assert that only the 32->64 switcher case should ever come here. */
4710 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4711 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4712 }
4713 break;
4714 }
4715
4716 default:
4717 {
4718 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4719 rc = VERR_INVALID_PARAMETER;
4720 break;
4721 }
4722 }
4723 AssertRCReturn(rc, rc);
4724 return rc;
4725}
4726
4727
4728/**
4729 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4730 * hosts (except darwin) for 64-bit guests.
4731 *
4732 * @param pVCpu Pointer to the VMCPU.
4733 * @param idxField The VMCS field encoding.
4734 * @param u64Val 16, 32 or 64 bits value.
4735 */
4736VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4737{
4738 AssertPtr(pVCpu);
4739 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4740
4741 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4742 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4743
4744 /* Make sure there are no duplicates. */
4745 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4746 {
4747 if (pCache->Write.aField[i] == idxField)
4748 {
4749 pCache->Write.aFieldVal[i] = u64Val;
4750 return VINF_SUCCESS;
4751 }
4752 }
4753
4754 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4755 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4756 pCache->Write.cValidEntries++;
4757 return VINF_SUCCESS;
4758}
4759
4760/* Enable later when the assembly code uses these as callbacks. */
4761#if 0
4762/*
4763 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4764 *
4765 * @param pVCpu Pointer to the VMCPU.
4766 * @param pCache Pointer to the VMCS cache.
4767 *
4768 * @remarks No-long-jump zone!!!
4769 */
4770VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4771{
4772 AssertPtr(pCache);
4773 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4774 {
4775 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4776 AssertRC(rc);
4777 }
4778 pCache->Write.cValidEntries = 0;
4779}
4780
4781
4782/**
4783 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
4784 *
4785 * @param pVCpu Pointer to the VMCPU.
4786 * @param pCache Pointer to the VMCS cache.
4787 *
4788 * @remarks No-long-jump zone!!!
4789 */
4790VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4791{
4792 AssertPtr(pCache);
4793 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4794 {
4795 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4796 AssertRC(rc);
4797 }
4798}
4799#endif
4800#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4801
4802
4803/**
4804 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4805 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4806 * timer.
4807 *
4808 * @returns VBox status code.
4809 * @param pVCpu Pointer to the VMCPU.
4810 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4811 * out-of-sync. Make sure to update the required fields
4812 * before using them.
4813 * @remarks No-long-jump zone!!!
4814 */
4815static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4816{
4817 int rc = VERR_INTERNAL_ERROR_5;
4818 bool fOffsettedTsc = false;
4819 PVM pVM = pVCpu->CTX_SUFF(pVM);
4820 if (pVM->hm.s.vmx.fUsePreemptTimer)
4821 {
4822 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4823
4824 /* Make sure the returned values have sane upper and lower boundaries. */
4825 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4826 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4827 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4828 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4829
4830 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4831 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4832 }
4833 else
4834 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4835
4836 if (fOffsettedTsc)
4837 {
4838 uint64_t u64CurTSC = ASMReadTSC();
4839 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4840 {
4841 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4842 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4843
4844 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4845 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4846 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4847 }
4848 else
4849 {
4850 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4851 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4852 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4853 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4854 }
4855 }
4856 else
4857 {
4858 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4859 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4860 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4861 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4862 }
4863}
4864
4865
4866/**
4867 * Determines if an exception is a contributory exception. Contributory
4868 * exceptions are ones which can cause double-faults. Page-fault is
4869 * intentionally not included here as it's a conditional contributory exception.
4870 *
4871 * @returns true if the exception is contributory, false otherwise.
4872 * @param uVector The exception vector.
4873 */
4874DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4875{
4876 switch (uVector)
4877 {
4878 case X86_XCPT_GP:
4879 case X86_XCPT_SS:
4880 case X86_XCPT_NP:
4881 case X86_XCPT_TS:
4882 case X86_XCPT_DE:
4883 return true;
4884 default:
4885 break;
4886 }
4887 return false;
4888}
4889
4890
4891/**
4892 * Sets an event as a pending event to be injected into the guest.
4893 *
4894 * @param pVCpu Pointer to the VMCPU.
4895 * @param u32IntrInfo The VM-entry interruption-information field.
4896 * @param cbInstr The VM-entry instruction length in bytes (for software
4897 * interrupts, exceptions and privileged software
4898 * exceptions).
4899 * @param u32ErrCode The VM-entry exception error code.
4900 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4901 * page-fault.
4902 *
4903 * @remarks Statistics counter assumes this is a guest event being injected or
4904 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4905 * always incremented.
4906 */
4907DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4908 RTGCUINTPTR GCPtrFaultAddress)
4909{
4910 Assert(!pVCpu->hm.s.Event.fPending);
4911 pVCpu->hm.s.Event.fPending = true;
4912 pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4913 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4914 pVCpu->hm.s.Event.cbInstr = cbInstr;
4915 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4916
4917 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4918}
4919
4920
4921/**
4922 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4923 *
4924 * @param pVCpu Pointer to the VMCPU.
4925 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4926 * out-of-sync. Make sure to update the required fields
4927 * before using them.
4928 */
4929DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4930{
4931 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
4932 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4933 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4934 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
4935}
4936
4937
4938/**
4939 * Handle a condition that occurred while delivering an event through the guest
4940 * IDT.
4941 *
4942 * @returns VBox status code (informational error codes included).
4943 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
4944 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4945 * continue execution of the guest which will delivery the #DF.
4946 * @retval VINF_EM_RESET if we detected a triple-fault condition.
4947 *
4948 * @param pVCpu Pointer to the VMCPU.
4949 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4950 * out-of-sync. Make sure to update the required fields
4951 * before using them.
4952 * @param pVmxTransient Pointer to the VMX transient structure.
4953 *
4954 * @remarks No-long-jump zone!!!
4955 */
4956static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4957{
4958 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4959 AssertRCReturn(rc, rc);
4960 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4961 {
4962 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4963 AssertRCReturn(rc, rc);
4964
4965 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4966 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4967 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4968
4969 typedef enum
4970 {
4971 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4972 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4973 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4974 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4975 } VMXREFLECTXCPT;
4976
4977 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4978 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4979 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4980 {
4981 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4982 {
4983 enmReflect = VMXREFLECTXCPT_XCPT;
4984#ifdef VBOX_STRICT
4985 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4986 && uExitVector == X86_XCPT_PF)
4987 {
4988 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4989 }
4990#endif
4991 if ( uExitVector == X86_XCPT_PF
4992 && uIdtVector == X86_XCPT_PF)
4993 {
4994 pVmxTransient->fVectoringPF = true;
4995 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4996 }
4997 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4998 && hmR0VmxIsContributoryXcpt(uExitVector)
4999 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5000 || uIdtVector == X86_XCPT_PF))
5001 {
5002 enmReflect = VMXREFLECTXCPT_DF;
5003 }
5004 else if (uIdtVector == X86_XCPT_DF)
5005 enmReflect = VMXREFLECTXCPT_TF;
5006 }
5007 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5008 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5009 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5010 {
5011 /*
5012 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5013 * (whatever they are) as they reoccur when restarting the instruction.
5014 */
5015 enmReflect = VMXREFLECTXCPT_XCPT;
5016 }
5017 }
5018 else
5019 {
5020 /*
5021 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5022 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5023 * original exception to the guest after handling the VM-exit.
5024 */
5025 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5026 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5027 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5028 {
5029 enmReflect = VMXREFLECTXCPT_XCPT;
5030 }
5031 }
5032
5033 switch (enmReflect)
5034 {
5035 case VMXREFLECTXCPT_XCPT:
5036 {
5037 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5038 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5039 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5040
5041 uint32_t u32ErrCode = 0;
5042 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5043 {
5044 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5045 AssertRCReturn(rc, rc);
5046 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5047 }
5048
5049 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5050 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5051 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5052 rc = VINF_SUCCESS;
5053 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5054 pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
5055
5056 break;
5057 }
5058
5059 case VMXREFLECTXCPT_DF:
5060 {
5061 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5062 rc = VINF_HM_DOUBLE_FAULT;
5063 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5064 pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
5065
5066 break;
5067 }
5068
5069 case VMXREFLECTXCPT_TF:
5070 {
5071 rc = VINF_EM_RESET;
5072 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5073 uExitVector));
5074 break;
5075 }
5076
5077 default:
5078 Assert(rc == VINF_SUCCESS);
5079 break;
5080 }
5081 }
5082 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5083 return rc;
5084}
5085
5086
5087/**
5088 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5089 *
5090 * @returns VBox status code.
5091 * @param pVCpu Pointer to the VMCPU.
5092 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5093 * out-of-sync. Make sure to update the required fields
5094 * before using them.
5095 *
5096 * @remarks No-long-jump zone!!!
5097 */
5098static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5099{
5100 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5101 {
5102 uint32_t uVal = 0;
5103 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5104 AssertRCReturn(rc, rc);
5105 uint32_t uShadow = 0;
5106 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5107 AssertRCReturn(rc, rc);
5108
5109 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5110 CPUMSetGuestCR0(pVCpu, uVal);
5111 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5112 }
5113 return VINF_SUCCESS;
5114}
5115
5116
5117/**
5118 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5119 *
5120 * @returns VBox status code.
5121 * @param pVCpu Pointer to the VMCPU.
5122 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5123 * out-of-sync. Make sure to update the required fields
5124 * before using them.
5125 *
5126 * @remarks No-long-jump zone!!!
5127 */
5128static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5129{
5130 int rc = VINF_SUCCESS;
5131 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5132 {
5133 uint32_t uVal = 0;
5134 uint32_t uShadow = 0;
5135 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5136 AssertRCReturn(rc, rc);
5137 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5138 AssertRCReturn(rc, rc);
5139
5140 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5141 CPUMSetGuestCR4(pVCpu, uVal);
5142 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5143 }
5144 return rc;
5145}
5146
5147
5148/**
5149 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5150 *
5151 * @returns VBox status code.
5152 * @param pVCpu Pointer to the VMCPU.
5153 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5154 * out-of-sync. Make sure to update the required fields
5155 * before using them.
5156 *
5157 * @remarks No-long-jump zone!!!
5158 */
5159static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5160{
5161 int rc = VINF_SUCCESS;
5162 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5163 {
5164 uint64_t u64Val = 0;
5165 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5166 AssertRCReturn(rc, rc);
5167
5168 pMixedCtx->rip = u64Val;
5169 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5170 }
5171 return rc;
5172}
5173
5174
5175/**
5176 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5177 *
5178 * @returns VBox status code.
5179 * @param pVCpu Pointer to the VMCPU.
5180 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5181 * out-of-sync. Make sure to update the required fields
5182 * before using them.
5183 *
5184 * @remarks No-long-jump zone!!!
5185 */
5186static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5187{
5188 int rc = VINF_SUCCESS;
5189 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5190 {
5191 uint64_t u64Val = 0;
5192 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5193 AssertRCReturn(rc, rc);
5194
5195 pMixedCtx->rsp = u64Val;
5196 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5197 }
5198 return rc;
5199}
5200
5201
5202/**
5203 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5204 *
5205 * @returns VBox status code.
5206 * @param pVCpu Pointer to the VMCPU.
5207 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5208 * out-of-sync. Make sure to update the required fields
5209 * before using them.
5210 *
5211 * @remarks No-long-jump zone!!!
5212 */
5213static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5214{
5215 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5216 {
5217 uint32_t uVal = 0;
5218 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5219 AssertRCReturn(rc, rc);
5220
5221 pMixedCtx->eflags.u32 = uVal;
5222 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5223 {
5224 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5225 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5226
5227 pMixedCtx->eflags.Bits.u1VM = 0;
5228 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5229 }
5230
5231 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5232 }
5233 return VINF_SUCCESS;
5234}
5235
5236
5237/**
5238 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5239 * guest-CPU context.
5240 */
5241DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5242{
5243 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5244 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5245 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5246 return rc;
5247}
5248
5249
5250/**
5251 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5252 * from the guest-state area in the VMCS.
5253 *
5254 * @param pVCpu Pointer to the VMCPU.
5255 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5256 * out-of-sync. Make sure to update the required fields
5257 * before using them.
5258 *
5259 * @remarks No-long-jump zone!!!
5260 */
5261static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5262{
5263 uint32_t uIntrState = 0;
5264 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5265 AssertRC(rc);
5266
5267 if (!uIntrState)
5268 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5269 else
5270 {
5271 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5272 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5273 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5274 AssertRC(rc);
5275 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5276 AssertRC(rc);
5277
5278 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5279 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5280 }
5281}
5282
5283
5284/**
5285 * Saves the guest's activity state.
5286 *
5287 * @returns VBox status code.
5288 * @param pVCpu Pointer to the VMCPU.
5289 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5290 * out-of-sync. Make sure to update the required fields
5291 * before using them.
5292 *
5293 * @remarks No-long-jump zone!!!
5294 */
5295static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5296{
5297 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5298 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5299 return VINF_SUCCESS;
5300}
5301
5302
5303/**
5304 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5305 * the current VMCS into the guest-CPU context.
5306 *
5307 * @returns VBox status code.
5308 * @param pVCpu Pointer to the VMCPU.
5309 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5310 * out-of-sync. Make sure to update the required fields
5311 * before using them.
5312 *
5313 * @remarks No-long-jump zone!!!
5314 */
5315static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5316{
5317 int rc = VINF_SUCCESS;
5318 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5319 {
5320 uint32_t u32Val = 0;
5321 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5322 pMixedCtx->SysEnter.cs = u32Val;
5323 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5324 }
5325
5326 uint64_t u64Val = 0;
5327 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5328 {
5329 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5330 pMixedCtx->SysEnter.eip = u64Val;
5331 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5332 }
5333 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5334 {
5335 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5336 pMixedCtx->SysEnter.esp = u64Val;
5337 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5338 }
5339 return rc;
5340}
5341
5342
5343/**
5344 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5345 * context.
5346 *
5347 * @returns VBox status code.
5348 * @param pVCpu Pointer to the VMCPU.
5349 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5350 * out-of-sync. Make sure to update the required fields
5351 * before using them.
5352 *
5353 * @remarks No-long-jump zone!!!
5354 */
5355static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5356{
5357 int rc = VINF_SUCCESS;
5358 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5359 {
5360 uint64_t u64Val = 0;
5361 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5362 pMixedCtx->fs.u64Base = u64Val;
5363 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5364 }
5365 return rc;
5366}
5367
5368
5369/**
5370 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5371 * context.
5372 *
5373 * @returns VBox status code.
5374 * @param pVCpu Pointer to the VMCPU.
5375 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5376 * out-of-sync. Make sure to update the required fields
5377 * before using them.
5378 *
5379 * @remarks No-long-jump zone!!!
5380 */
5381static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5382{
5383 int rc = VINF_SUCCESS;
5384 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5385 {
5386 uint64_t u64Val = 0;
5387 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5388 pMixedCtx->gs.u64Base = u64Val;
5389 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5390 }
5391 return rc;
5392}
5393
5394
5395/**
5396 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5397 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5398 * and TSC_AUX.
5399 *
5400 * @returns VBox status code.
5401 * @param pVCpu Pointer to the VMCPU.
5402 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5403 * out-of-sync. Make sure to update the required fields
5404 * before using them.
5405 *
5406 * @remarks No-long-jump zone!!!
5407 */
5408static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5409{
5410 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5411 return VINF_SUCCESS;
5412
5413#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5414 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5415 {
5416 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5417 pMsr += i;
5418 switch (pMsr->u32Msr)
5419 {
5420 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5421 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5422 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5423 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5424 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5425 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5426 default:
5427 {
5428 AssertFailed();
5429 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5430 }
5431 }
5432 }
5433#endif
5434
5435 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5436 return VINF_SUCCESS;
5437}
5438
5439
5440/**
5441 * Saves the guest control registers from the current VMCS into the guest-CPU
5442 * context.
5443 *
5444 * @returns VBox status code.
5445 * @param pVCpu Pointer to the VMCPU.
5446 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5447 * out-of-sync. Make sure to update the required fields
5448 * before using them.
5449 *
5450 * @remarks No-long-jump zone!!!
5451 */
5452static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5453{
5454 /* Guest CR0. Guest FPU. */
5455 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5456 AssertRCReturn(rc, rc);
5457
5458 /* Guest CR4. */
5459 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5460 AssertRCReturn(rc, rc);
5461
5462 /* Guest CR2 - updated always during the world-switch or in #PF. */
5463 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5464 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5465 {
5466 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5467 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5468
5469 PVM pVM = pVCpu->CTX_SUFF(pVM);
5470 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5471 || ( pVM->hm.s.fNestedPaging
5472 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5473 {
5474 uint64_t u64Val = 0;
5475 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5476 if (pMixedCtx->cr3 != u64Val)
5477 {
5478 CPUMSetGuestCR3(pVCpu, u64Val);
5479 if (VMMRZCallRing3IsEnabled(pVCpu))
5480 {
5481 PGMUpdateCR3(pVCpu, u64Val);
5482 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5483 }
5484 else
5485 {
5486 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5487 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5488 }
5489 }
5490
5491 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5492 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5493 {
5494 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5495 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5496 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5497 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5498
5499 if (VMMRZCallRing3IsEnabled(pVCpu))
5500 {
5501 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5502 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5503 }
5504 else
5505 {
5506 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5507 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5508 }
5509 }
5510 }
5511
5512 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5513 }
5514
5515 /*
5516 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5517 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5518 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5519 *
5520 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5521 */
5522 if (VMMRZCallRing3IsEnabled(pVCpu))
5523 {
5524 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5525 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5526
5527 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5528 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5529
5530 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5531 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5532 }
5533
5534 return rc;
5535}
5536
5537
5538/**
5539 * Reads a guest segment register from the current VMCS into the guest-CPU
5540 * context.
5541 *
5542 * @returns VBox status code.
5543 * @param pVCpu Pointer to the VMCPU.
5544 * @param idxSel Index of the selector in the VMCS.
5545 * @param idxLimit Index of the segment limit in the VMCS.
5546 * @param idxBase Index of the segment base in the VMCS.
5547 * @param idxAccess Index of the access rights of the segment in the VMCS.
5548 * @param pSelReg Pointer to the segment selector.
5549 *
5550 * @remarks No-long-jump zone!!!
5551 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5552 * macro as that takes care of whether to read from the VMCS cache or
5553 * not.
5554 */
5555DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5556 PCPUMSELREG pSelReg)
5557{
5558 uint32_t u32Val = 0;
5559 int rc = VMXReadVmcs32(idxSel, &u32Val);
5560 AssertRCReturn(rc, rc);
5561 pSelReg->Sel = (uint16_t)u32Val;
5562 pSelReg->ValidSel = (uint16_t)u32Val;
5563 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5564
5565 rc = VMXReadVmcs32(idxLimit, &u32Val);
5566 AssertRCReturn(rc, rc);
5567 pSelReg->u32Limit = u32Val;
5568
5569 uint64_t u64Val = 0;
5570 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5571 AssertRCReturn(rc, rc);
5572 pSelReg->u64Base = u64Val;
5573
5574 rc = VMXReadVmcs32(idxAccess, &u32Val);
5575 AssertRCReturn(rc, rc);
5576 pSelReg->Attr.u = u32Val;
5577
5578 /*
5579 * If VT-x marks the segment as unusable, most other bits remain undefined:
5580 * - For CS the L, D and G bits have meaning.
5581 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5582 * - For the remaining data segments no bits are defined.
5583 *
5584 * The present bit and the unusable bit has been observed to be set at the
5585 * same time (the selector was supposed to invalid as we started executing
5586 * a V8086 interrupt in ring-0).
5587 *
5588 * What should be important for the rest of the VBox code that the P bit is
5589 * cleared. Some of the other VBox code recognizes the unusable bit, but
5590 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5591 * safe side here, we'll strip off P and other bits we don't care about. If
5592 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5593 *
5594 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5595 */
5596 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5597 {
5598 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5599
5600 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5601 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5602 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5603
5604 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5605#ifdef DEBUG_bird
5606 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5607 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5608 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5609#endif
5610 }
5611 return VINF_SUCCESS;
5612}
5613
5614
5615#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5616# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5617 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5618 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5619#else
5620# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5621 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5622 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5623#endif
5624
5625
5626/**
5627 * Saves the guest segment registers from the current VMCS into the guest-CPU
5628 * context.
5629 *
5630 * @returns VBox status code.
5631 * @param pVCpu Pointer to the VMCPU.
5632 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5633 * out-of-sync. Make sure to update the required fields
5634 * before using them.
5635 *
5636 * @remarks No-long-jump zone!!!
5637 */
5638static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5639{
5640 /* Guest segment registers. */
5641 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5642 {
5643 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5644 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5645 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5646 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5647 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5648 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5649 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5650
5651 /* Restore segment attributes for real-on-v86 mode hack. */
5652 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5653 {
5654 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5655 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5656 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5657 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5658 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5659 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5660 }
5661 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5662 }
5663
5664 return VINF_SUCCESS;
5665}
5666
5667
5668/**
5669 * Saves the guest descriptor table registers and task register from the current
5670 * VMCS into the guest-CPU context.
5671 *
5672 * @returns VBox status code.
5673 * @param pVCpu Pointer to the VMCPU.
5674 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5675 * out-of-sync. Make sure to update the required fields
5676 * before using them.
5677 *
5678 * @remarks No-long-jump zone!!!
5679 */
5680static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5681{
5682 int rc = VINF_SUCCESS;
5683
5684 /* Guest LDTR. */
5685 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5686 {
5687 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5688 AssertRCReturn(rc, rc);
5689 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5690 }
5691
5692 /* Guest GDTR. */
5693 uint64_t u64Val = 0;
5694 uint32_t u32Val = 0;
5695 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5696 {
5697 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5698 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5699 pMixedCtx->gdtr.pGdt = u64Val;
5700 pMixedCtx->gdtr.cbGdt = u32Val;
5701 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
5702 }
5703
5704 /* Guest IDTR. */
5705 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5706 {
5707 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5708 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5709 pMixedCtx->idtr.pIdt = u64Val;
5710 pMixedCtx->idtr.cbIdt = u32Val;
5711 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
5712 }
5713
5714 /* Guest TR. */
5715 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5716 {
5717 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5718 AssertRCReturn(rc, rc);
5719
5720 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5721 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5722 {
5723 rc = VMXLOCAL_READ_SEG(TR, tr);
5724 AssertRCReturn(rc, rc);
5725 }
5726 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
5727 }
5728 return rc;
5729}
5730
5731#undef VMXLOCAL_READ_SEG
5732
5733
5734/**
5735 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5736 * context.
5737 *
5738 * @returns VBox status code.
5739 * @param pVCpu Pointer to the VMCPU.
5740 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5741 * out-of-sync. Make sure to update the required fields
5742 * before using them.
5743 *
5744 * @remarks No-long-jump zone!!!
5745 */
5746static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5747{
5748 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5749 {
5750 if (!CPUMIsHyperDebugStateActive(pVCpu))
5751 {
5752 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5753 uint32_t u32Val;
5754 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5755 pMixedCtx->dr[7] = u32Val;
5756 }
5757
5758 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
5759 }
5760 return VINF_SUCCESS;
5761}
5762
5763
5764/**
5765 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
5766 *
5767 * @returns VBox status code.
5768 * @param pVCpu Pointer to the VMCPU.
5769 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5770 * out-of-sync. Make sure to update the required fields
5771 * before using them.
5772 *
5773 * @remarks No-long-jump zone!!!
5774 */
5775static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5776{
5777 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5778 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
5779 return VINF_SUCCESS;
5780}
5781
5782
5783/**
5784 * Saves the entire guest state from the currently active VMCS into the
5785 * guest-CPU context. This essentially VMREADs all guest-data.
5786 *
5787 * @returns VBox status code.
5788 * @param pVCpu Pointer to the VMCPU.
5789 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5790 * out-of-sync. Make sure to update the required fields
5791 * before using them.
5792 */
5793static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5794{
5795 Assert(pVCpu);
5796 Assert(pMixedCtx);
5797
5798 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5799 return VINF_SUCCESS;
5800
5801 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5802 again on the ring-3 callback path, there is no real need to. */
5803 if (VMMRZCallRing3IsEnabled(pVCpu))
5804 VMMR0LogFlushDisable(pVCpu);
5805 else
5806 Assert(VMMR0IsLogFlushDisabled(pVCpu));
5807 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5808
5809 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5810 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5811
5812 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5813 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5814
5815 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5816 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5817
5818 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5819 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5820
5821 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5822 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5823
5824 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5825 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5826
5827 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5828 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5829
5830 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5831 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5832
5833 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5834 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5835
5836 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5837 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5838
5839 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5840 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5841
5842 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5843 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5844
5845 if (VMMRZCallRing3IsEnabled(pVCpu))
5846 VMMR0LogFlushEnable(pVCpu);
5847
5848 return rc;
5849}
5850
5851
5852/**
5853 * Check per-VM and per-VCPU force flag actions that require us to go back to
5854 * ring-3 for one reason or another.
5855 *
5856 * @returns VBox status code (information status code included).
5857 * @retval VINF_SUCCESS if we don't have any actions that require going back to
5858 * ring-3.
5859 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5860 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5861 * interrupts)
5862 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5863 * all EMTs to be in ring-3.
5864 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5865 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5866 * to the EM loop.
5867 *
5868 * @param pVM Pointer to the VM.
5869 * @param pVCpu Pointer to the VMCPU.
5870 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5871 * out-of-sync. Make sure to update the required fields
5872 * before using them.
5873 */
5874static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5875{
5876 Assert(VMMRZCallRing3IsEnabled(pVCpu));
5877
5878 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5879 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5880 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5881 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5882 {
5883 /* We need the control registers now, make sure the guest-CPU context is updated. */
5884 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5885 AssertRCReturn(rc3, rc3);
5886
5887 /* Pending HM CR3 sync. */
5888 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5889 {
5890 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5891 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
5892 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5893 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5894 }
5895
5896 /* Pending HM PAE PDPEs. */
5897 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5898 {
5899 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5900 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5901 }
5902
5903 /* Pending PGM C3 sync. */
5904 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5905 {
5906 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
5907 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5908 if (rc2 != VINF_SUCCESS)
5909 {
5910 AssertRC(rc2);
5911 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5912 return rc2;
5913 }
5914 }
5915
5916 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5917 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5918 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5919 {
5920 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5921 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5922 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5923 return rc2;
5924 }
5925
5926 /* Pending VM request packets, such as hardware interrupts. */
5927 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5928 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5929 {
5930 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5931 return VINF_EM_PENDING_REQUEST;
5932 }
5933
5934 /* Pending PGM pool flushes. */
5935 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5936 {
5937 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5938 return VINF_PGM_POOL_FLUSH_PENDING;
5939 }
5940
5941 /* Pending DMA requests. */
5942 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5943 {
5944 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5945 return VINF_EM_RAW_TO_R3;
5946 }
5947 }
5948
5949 return VINF_SUCCESS;
5950}
5951
5952
5953/**
5954 * Converts any TRPM trap into a pending HM event. This is typically used when
5955 * entering from ring-3 (not longjmp returns).
5956 *
5957 * @param pVCpu Pointer to the VMCPU.
5958 */
5959static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5960{
5961 Assert(TRPMHasTrap(pVCpu));
5962 Assert(!pVCpu->hm.s.Event.fPending);
5963
5964 uint8_t uVector;
5965 TRPMEVENT enmTrpmEvent;
5966 RTGCUINT uErrCode;
5967 RTGCUINTPTR GCPtrFaultAddress;
5968 uint8_t cbInstr;
5969
5970 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5971 AssertRC(rc);
5972
5973 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5974 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
5975 if (enmTrpmEvent == TRPM_TRAP)
5976 {
5977 switch (uVector)
5978 {
5979 case X86_XCPT_BP:
5980 case X86_XCPT_OF:
5981 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5982 break;
5983
5984 case X86_XCPT_PF:
5985 case X86_XCPT_DF:
5986 case X86_XCPT_TS:
5987 case X86_XCPT_NP:
5988 case X86_XCPT_SS:
5989 case X86_XCPT_GP:
5990 case X86_XCPT_AC:
5991 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5992 /* no break! */
5993 default:
5994 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5995 break;
5996 }
5997 }
5998 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5999 {
6000 if (uVector == X86_XCPT_NMI)
6001 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6002 else
6003 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6004 }
6005 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6006 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6007 else
6008 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6009
6010 rc = TRPMResetTrap(pVCpu);
6011 AssertRC(rc);
6012 Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6013 u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6014
6015 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6016 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6017}
6018
6019
6020/**
6021 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6022 * VT-x to execute any instruction.
6023 *
6024 * @param pvCpu Pointer to the VMCPU.
6025 */
6026static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6027{
6028 Assert(pVCpu->hm.s.Event.fPending);
6029
6030 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6031 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
6032 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
6033 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6034
6035 /* If a trap was already pending, we did something wrong! */
6036 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6037
6038 TRPMEVENT enmTrapType;
6039 switch (uVectorType)
6040 {
6041 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6042 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6043 enmTrapType = TRPM_HARDWARE_INT;
6044 break;
6045
6046 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6047 enmTrapType = TRPM_SOFTWARE_INT;
6048 break;
6049
6050 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6051 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6052 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6053 enmTrapType = TRPM_TRAP;
6054 break;
6055
6056 default:
6057 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6058 enmTrapType = TRPM_32BIT_HACK;
6059 break;
6060 }
6061
6062 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6063
6064 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6065 AssertRC(rc);
6066
6067 if (fErrorCodeValid)
6068 TRPMSetErrorCode(pVCpu, uErrorCode);
6069
6070 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6071 && uVector == X86_XCPT_PF)
6072 {
6073 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6074 }
6075 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6076 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6077 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6078 {
6079 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6080 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6081 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6082 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6083 }
6084 pVCpu->hm.s.Event.fPending = false;
6085}
6086
6087
6088/**
6089 * Does the necessary state syncing before returning to ring-3 for any reason
6090 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6091 *
6092 * @returns VBox status code.
6093 * @param pVM Pointer to the VM.
6094 * @param pVCpu Pointer to the VMCPU.
6095 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6096 * be out-of-sync. Make sure to update the required
6097 * fields before using them.
6098 * @param fSaveGuestState Whether to save the guest state or not.
6099 *
6100 * @remarks No-long-jmp zone!!!
6101 */
6102static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6103{
6104 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6105 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6106
6107 RTCPUID idCpu = RTMpCpuId();
6108 Log4Func(("HostCpuId=%u\n", idCpu));
6109
6110 /* Save the guest state if necessary. */
6111 if ( fSaveGuestState
6112 && pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6113 {
6114 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6115 AssertRCReturn(rc, rc);
6116 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6117 }
6118
6119 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6120 if (CPUMIsGuestFPUStateActive(pVCpu))
6121 {
6122 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6123 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6124 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
6125 }
6126
6127 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6128#ifdef VBOX_STRICT
6129 if (CPUMIsHyperDebugStateActive(pVCpu))
6130 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6131#endif
6132 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6133 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
6134 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6135 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6136
6137#if HC_ARCH_BITS == 64
6138 /* Restore host-state bits that VT-x only restores partially. */
6139 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6140 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6141 {
6142 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6143 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6144 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6145 }
6146#endif
6147
6148 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6149 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6150 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6151 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6152 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6153 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6154 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6155 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6156
6157 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6158
6159 /** @todo This kinda defeats the purpose of having preemption hooks.
6160 * The problem is, deregistering the hooks should be moved to a place that
6161 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6162 * context.
6163 */
6164 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6165 {
6166 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6167 AssertRCReturn(rc, rc);
6168
6169 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6170 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6171 }
6172 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6173 NOREF(idCpu);
6174
6175 return VINF_SUCCESS;
6176}
6177
6178
6179/**
6180 * Leaves the VT-x session.
6181 *
6182 * @returns VBox status code.
6183 * @param pVM Pointer to the VM.
6184 * @param pVCpu Pointer to the VMCPU.
6185 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6186 * out-of-sync. Make sure to update the required fields
6187 * before using them.
6188 *
6189 * @remarks No-long-jmp zone!!!
6190 */
6191DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6192{
6193 HM_DISABLE_PREEMPT_IF_NEEDED();
6194 HMVMX_ASSERT_CPU_SAFE();
6195 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6196 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6197
6198 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6199 and done this from the VMXR0ThreadCtxCallback(). */
6200 if (!pVCpu->hm.s.fLeaveDone)
6201 {
6202 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6203 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6204 pVCpu->hm.s.fLeaveDone = true;
6205 }
6206
6207 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6208 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6209 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6210 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6211 VMMR0ThreadCtxHooksDeregister(pVCpu);
6212
6213 /* Leave HM context. This takes care of local init (term). */
6214 int rc = HMR0LeaveCpu(pVCpu);
6215
6216 HM_RESTORE_PREEMPT_IF_NEEDED();
6217
6218 return rc;
6219}
6220
6221
6222/**
6223 * Does the necessary state syncing before doing a longjmp to ring-3.
6224 *
6225 * @returns VBox status code.
6226 * @param pVM Pointer to the VM.
6227 * @param pVCpu Pointer to the VMCPU.
6228 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6229 * out-of-sync. Make sure to update the required fields
6230 * before using them.
6231 *
6232 * @remarks No-long-jmp zone!!!
6233 */
6234DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6235{
6236 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6237}
6238
6239
6240/**
6241 * Take necessary actions before going back to ring-3.
6242 *
6243 * An action requires us to go back to ring-3. This function does the necessary
6244 * steps before we can safely return to ring-3. This is not the same as longjmps
6245 * to ring-3, this is voluntary and prepares the guest so it may continue
6246 * executing outside HM (recompiler/IEM).
6247 *
6248 * @returns VBox status code.
6249 * @param pVM Pointer to the VM.
6250 * @param pVCpu Pointer to the VMCPU.
6251 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6252 * out-of-sync. Make sure to update the required fields
6253 * before using them.
6254 * @param rcExit The reason for exiting to ring-3. Can be
6255 * VINF_VMM_UNKNOWN_RING3_CALL.
6256 */
6257static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6258{
6259 Assert(pVM);
6260 Assert(pVCpu);
6261 Assert(pMixedCtx);
6262 HMVMX_ASSERT_PREEMPT_SAFE();
6263
6264 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6265 {
6266 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6267 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6268 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6269 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6270 }
6271
6272 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6273 VMMRZCallRing3Disable(pVCpu);
6274 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6275
6276 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6277 if (pVCpu->hm.s.Event.fPending)
6278 {
6279 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6280 Assert(!pVCpu->hm.s.Event.fPending);
6281 }
6282
6283 /* Save guest state and restore host state bits. */
6284 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6285 AssertRCReturn(rc, rc);
6286 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6287
6288 /* Sync recompiler state. */
6289 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6290 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6291 | CPUM_CHANGED_LDTR
6292 | CPUM_CHANGED_GDTR
6293 | CPUM_CHANGED_IDTR
6294 | CPUM_CHANGED_TR
6295 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6296 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6297 if ( pVM->hm.s.fNestedPaging
6298 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6299 {
6300 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6301 }
6302
6303 /*
6304 * Clear the X86_EFL_TF if necessary.
6305 */
6306 if (pVCpu->hm.s.fClearTrapFlag)
6307 {
6308 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6309 pMixedCtx->eflags.Bits.u1TF = 0;
6310 pVCpu->hm.s.fClearTrapFlag = false;
6311 }
6312 /** @todo there seems to be issues with the resume flag when the monitor trap
6313 * flag is pending without being used. Seen early in bios init when
6314 * accessing APIC page in prot mode. */
6315
6316 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6317 if (rcExit != VINF_EM_RAW_INTERRUPT)
6318 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
6319
6320 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6321
6322 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6323 VMMRZCallRing3RemoveNotification(pVCpu);
6324 VMMRZCallRing3Enable(pVCpu);
6325
6326 return rc;
6327}
6328
6329
6330/**
6331 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6332 * longjump to ring-3 and possibly get preempted.
6333 *
6334 * @returns VBox status code.
6335 * @param pVCpu Pointer to the VMCPU.
6336 * @param enmOperation The operation causing the ring-3 longjump.
6337 * @param pvUser Opaque pointer to the guest-CPU context. The data
6338 * may be out-of-sync. Make sure to update the required
6339 * fields before using them.
6340 *
6341 * @remarks Must never be called with @a enmOperation ==
6342 * VMMCALLRING3_VM_R0_ASSERTION. We can't assert it here because if it
6343 * it -does- get called with VMMCALLRING3_VM_R0_ASSERTION, we'll end up
6344 * with an infinite recursion.
6345 */
6346DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6347{
6348 /* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6349 Assert(pVCpu);
6350 Assert(pvUser);
6351 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6352 HMVMX_ASSERT_PREEMPT_SAFE();
6353
6354 VMMRZCallRing3Disable(pVCpu);
6355 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6356
6357 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6358 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6359 AssertRCReturn(rc, rc);
6360
6361 VMMRZCallRing3Enable(pVCpu);
6362 return VINF_SUCCESS;
6363}
6364
6365
6366/**
6367 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6368 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6369 *
6370 * @param pVCpu Pointer to the VMCPU.
6371 */
6372DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6373{
6374 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6375 {
6376 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6377 {
6378 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6379 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6380 AssertRC(rc);
6381 }
6382 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6383}
6384
6385
6386/**
6387 * Evaluates the event to be delivered to the guest and sets it as the pending
6388 * event.
6389 *
6390 * @param pVCpu Pointer to the VMCPU.
6391 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6392 * out-of-sync. Make sure to update the required fields
6393 * before using them.
6394 */
6395static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6396{
6397 Assert(!pVCpu->hm.s.Event.fPending);
6398
6399 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6400 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6401 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6402 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6403
6404 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6405 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6406 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6407 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6408 Assert(!TRPMHasTrap(pVCpu));
6409
6410 /** @todo SMI. SMIs take priority over NMIs. */
6411 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6412 {
6413 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6414 if ( !fBlockMovSS
6415 && !fBlockSti)
6416 {
6417 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6418 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6419 uint32_t u32IntrInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6420 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6421
6422 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6423 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6424 }
6425 else
6426 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6427 }
6428 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6429 && !pVCpu->hm.s.fSingleInstruction)
6430 {
6431 /*
6432 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6433 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6434 * evaluated here and not set as pending, solely based on the force-flags.
6435 */
6436 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6437 AssertRC(rc);
6438 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6439 if ( !fBlockInt
6440 && !fBlockSti
6441 && !fBlockMovSS)
6442 {
6443 uint8_t u8Interrupt;
6444 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6445 if (RT_SUCCESS(rc))
6446 {
6447 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6448 uint32_t u32IntrInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6449 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6450
6451 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6452 }
6453 else
6454 {
6455 /** @todo Does this actually happen? If not turn it into an assertion. */
6456 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6457 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6458 }
6459 }
6460 else
6461 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6462 }
6463}
6464
6465
6466/**
6467 * Injects any pending events into the guest if the guest is in a state to
6468 * receive them.
6469 *
6470 * @returns VBox status code (informational status codes included).
6471 * @param pVCpu Pointer to the VMCPU.
6472 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6473 * out-of-sync. Make sure to update the required fields
6474 * before using them.
6475 */
6476static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6477{
6478 HMVMX_ASSERT_PREEMPT_SAFE();
6479 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6480
6481 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6482 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6483 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6484 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6485
6486 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6487 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6488 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6489 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6490 Assert(!TRPMHasTrap(pVCpu));
6491
6492 int rc = VINF_SUCCESS;
6493 if (pVCpu->hm.s.Event.fPending)
6494 {
6495#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6496 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6497 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6498 {
6499 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6500 AssertRCReturn(rc, rc);
6501 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6502 Assert(!fBlockInt);
6503 Assert(!fBlockSti);
6504 Assert(!fBlockMovSS);
6505 }
6506 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6507 {
6508 Assert(!fBlockSti);
6509 Assert(!fBlockMovSS);
6510 }
6511#endif
6512 Log4(("Injecting pending event vcpu[%RU32] u64IntrInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo));
6513 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6514 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6515 AssertRCReturn(rc, rc);
6516
6517 /* Update the interruptibility-state as it could have been changed by
6518 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6519 fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6520 fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6521
6522#ifdef VBOX_WITH_STATISTICS
6523 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6524 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6525 else
6526 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6527#endif
6528 }
6529
6530 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6531 int rc2 = VINF_SUCCESS;
6532 if ( fBlockSti
6533 || fBlockMovSS)
6534 {
6535 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6536 {
6537 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6538 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6539 {
6540 /*
6541 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6542 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6543 * See Intel spec. 27.3.4 "Saving Non-Register State".
6544 */
6545 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6546 AssertRCReturn(rc, rc);
6547 }
6548 }
6549 else
6550 {
6551 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6552 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6553 uIntrState = 0;
6554 }
6555 }
6556
6557 /*
6558 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6559 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6560 */
6561 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6562 AssertRC(rc2);
6563
6564 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6565 return rc;
6566}
6567
6568
6569/**
6570 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6571 *
6572 * @param pVCpu Pointer to the VMCPU.
6573 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6574 * out-of-sync. Make sure to update the required fields
6575 * before using them.
6576 */
6577DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6578{
6579 uint32_t u32IntrInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6580 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6581}
6582
6583
6584/**
6585 * Injects a double-fault (#DF) exception into the VM.
6586 *
6587 * @returns VBox status code (informational status code included).
6588 * @param pVCpu Pointer to the VMCPU.
6589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6590 * out-of-sync. Make sure to update the required fields
6591 * before using them.
6592 */
6593DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6594{
6595 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6596 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6597 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6598 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6599 puIntrState);
6600}
6601
6602
6603/**
6604 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6605 *
6606 * @param pVCpu Pointer to the VMCPU.
6607 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6608 * out-of-sync. Make sure to update the required fields
6609 * before using them.
6610 */
6611DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6612{
6613 uint32_t u32IntrInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6614 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6615 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6616}
6617
6618
6619/**
6620 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6621 *
6622 * @param pVCpu Pointer to the VMCPU.
6623 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6624 * out-of-sync. Make sure to update the required fields
6625 * before using them.
6626 * @param cbInstr The value of RIP that is to be pushed on the guest
6627 * stack.
6628 */
6629DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6630{
6631 uint32_t u32IntrInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6632 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6633 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6634}
6635
6636
6637/**
6638 * Injects a general-protection (#GP) fault into the VM.
6639 *
6640 * @returns VBox status code (informational status code included).
6641 * @param pVCpu Pointer to the VMCPU.
6642 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6643 * out-of-sync. Make sure to update the required fields
6644 * before using them.
6645 * @param u32ErrorCode The error code associated with the #GP.
6646 */
6647DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6648 uint32_t *puIntrState)
6649{
6650 uint32_t u32IntrInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6651 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6652 if (fErrorCodeValid)
6653 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6654 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6655 puIntrState);
6656}
6657
6658
6659/**
6660 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6661 *
6662 * @param pVCpu Pointer to the VMCPU.
6663 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6664 * out-of-sync. Make sure to update the required fields
6665 * before using them.
6666 * @param uVector The software interrupt vector number.
6667 * @param cbInstr The value of RIP that is to be pushed on the guest
6668 * stack.
6669 */
6670DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6671{
6672 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6673 if ( uVector == X86_XCPT_BP
6674 || uVector == X86_XCPT_OF)
6675 {
6676 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6677 }
6678 else
6679 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6680 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6681}
6682
6683
6684/**
6685 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6686 * stack.
6687 *
6688 * @returns VBox status code (information status code included).
6689 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6690 * @param pVM Pointer to the VM.
6691 * @param pMixedCtx Pointer to the guest-CPU context.
6692 * @param uValue The value to push to the guest stack.
6693 */
6694DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6695{
6696 /*
6697 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6698 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6699 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6700 */
6701 if (pMixedCtx->sp == 1)
6702 return VINF_EM_RESET;
6703 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6704 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6705 AssertRCReturn(rc, rc);
6706 return rc;
6707}
6708
6709
6710/**
6711 * Injects an event into the guest upon VM-entry by updating the relevant fields
6712 * in the VM-entry area in the VMCS.
6713 *
6714 * @returns VBox status code (informational error codes included).
6715 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6716 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6717 *
6718 * @param pVCpu Pointer to the VMCPU.
6719 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6720 * be out-of-sync. Make sure to update the required
6721 * fields before using them.
6722 * @param u64IntrInfo The VM-entry interruption-information field.
6723 * @param cbInstr The VM-entry instruction length in bytes (for
6724 * software interrupts, exceptions and privileged
6725 * software exceptions).
6726 * @param u32ErrCode The VM-entry exception error code.
6727 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6728 * @param puIntrState Pointer to the current guest interruptibility-state.
6729 * This interruptibility-state will be updated if
6730 * necessary. This cannot not be NULL.
6731 *
6732 * @remarks Requires CR0!
6733 * @remarks No-long-jump zone!!!
6734 */
6735static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6736 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6737{
6738 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6739 AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6740 Assert(puIntrState);
6741 uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6742
6743 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6744 const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6745
6746#ifdef VBOX_STRICT
6747 /* Validate the error-code-valid bit for hardware exceptions. */
6748 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6749 {
6750 switch (uVector)
6751 {
6752 case X86_XCPT_PF:
6753 case X86_XCPT_DF:
6754 case X86_XCPT_TS:
6755 case X86_XCPT_NP:
6756 case X86_XCPT_SS:
6757 case X86_XCPT_GP:
6758 case X86_XCPT_AC:
6759 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6760 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6761 /* fallthru */
6762 default:
6763 break;
6764 }
6765 }
6766#endif
6767
6768 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
6769 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6770 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6771
6772 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6773
6774 /* We require CR0 to check if the guest is in real-mode. */
6775 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6776 AssertRCReturn(rc, rc);
6777
6778 /*
6779 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6780 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6781 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6782 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6783 */
6784 if (CPUMIsGuestInRealModeEx(pMixedCtx))
6785 {
6786 PVM pVM = pVCpu->CTX_SUFF(pVM);
6787 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6788 {
6789 Assert(PDMVmmDevHeapIsEnabled(pVM));
6790 Assert(pVM->hm.s.vmx.pRealModeTSS);
6791
6792 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6793 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6794 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6795 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6796 AssertRCReturn(rc, rc);
6797 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6798
6799 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6800 const size_t cbIdtEntry = sizeof(X86IDTR16);
6801 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6802 {
6803 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6804 if (uVector == X86_XCPT_DF)
6805 return VINF_EM_RESET;
6806 else if (uVector == X86_XCPT_GP)
6807 {
6808 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6809 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6810 }
6811
6812 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6813 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6814 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
6815 }
6816
6817 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6818 uint16_t uGuestIp = pMixedCtx->ip;
6819 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6820 {
6821 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
6822 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
6823 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6824 }
6825 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6826 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6827
6828 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6829 X86IDTR16 IdtEntry;
6830 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6831 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6832 AssertRCReturn(rc, rc);
6833
6834 /* Construct the stack frame for the interrupt/exception handler. */
6835 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6836 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6837 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6838 AssertRCReturn(rc, rc);
6839
6840 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
6841 if (rc == VINF_SUCCESS)
6842 {
6843 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
6844 pMixedCtx->rip = IdtEntry.offSel;
6845 pMixedCtx->cs.Sel = IdtEntry.uSel;
6846 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6847 if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6848 && uVector == X86_XCPT_PF)
6849 {
6850 pMixedCtx->cr2 = GCPtrFaultAddress;
6851 }
6852
6853 /* If any other guest-state bits are changed here, make sure to update
6854 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6855 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS
6856 | HM_CHANGED_GUEST_RIP
6857 | HM_CHANGED_GUEST_RFLAGS
6858 | HM_CHANGED_GUEST_RSP;
6859
6860 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6861 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6862 {
6863 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6864 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6865 Log4(("Clearing inhibition due to STI.\n"));
6866 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6867 }
6868 Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6869
6870 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
6871 it, if we are returning to ring-3 before executing guest code. */
6872 pVCpu->hm.s.Event.fPending = false;
6873 }
6874 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6875 return rc;
6876 }
6877 else
6878 {
6879 /*
6880 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6881 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6882 */
6883 u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6884 }
6885 }
6886
6887 /* Validate. */
6888 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6889 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6890 Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6891
6892 /* Inject. */
6893 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6894 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6895 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6896 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6897
6898 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6899 && uVector == X86_XCPT_PF)
6900 {
6901 pMixedCtx->cr2 = GCPtrFaultAddress;
6902 }
6903
6904 Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6905 u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6906
6907 AssertRCReturn(rc, rc);
6908 return rc;
6909}
6910
6911
6912/**
6913 * Clears the interrupt-window exiting control in the VMCS and if necessary
6914 * clears the current event in the VMCS as well.
6915 *
6916 * @returns VBox status code.
6917 * @param pVCpu Pointer to the VMCPU.
6918 *
6919 * @remarks Use this function only to clear events that have not yet been
6920 * delivered to the guest but are injected in the VMCS!
6921 * @remarks No-long-jump zone!!!
6922 */
6923static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
6924{
6925 int rc;
6926
6927 /* Clear interrupt-window exiting control. */
6928 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
6929 {
6930 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6931 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6932 AssertRC(rc);
6933 }
6934
6935 if (!pVCpu->hm.s.Event.fPending)
6936 return;
6937
6938#ifdef VBOX_STRICT
6939 uint32_t u32EntryInfo;
6940 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
6941 AssertRC(rc);
6942 Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
6943#endif
6944
6945 /* Clear the entry-interruption field (including the valid bit). */
6946 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
6947 AssertRC(rc);
6948
6949 /* Clear the pending debug exception field. */
6950 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
6951 AssertRC(rc);
6952}
6953
6954
6955/**
6956 * Enters the VT-x session.
6957 *
6958 * @returns VBox status code.
6959 * @param pVM Pointer to the VM.
6960 * @param pVCpu Pointer to the VMCPU.
6961 * @param pCpu Pointer to the CPU info struct.
6962 */
6963VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
6964{
6965 AssertPtr(pVM);
6966 AssertPtr(pVCpu);
6967 Assert(pVM->hm.s.vmx.fSupported);
6968 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6969 NOREF(pCpu);
6970
6971 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6972 Assert((pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE))
6973 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
6974
6975#ifdef VBOX_STRICT
6976 /* Make sure we're in VMX root mode. */
6977 RTCCUINTREG u32HostCR4 = ASMGetCR4();
6978 if (!(u32HostCR4 & X86_CR4_VMXE))
6979 {
6980 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6981 return VERR_VMX_X86_CR4_VMXE_CLEARED;
6982 }
6983#endif
6984
6985 /*
6986 * Load the VCPU's VMCS as the current (and active) one.
6987 */
6988 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
6989 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6990 if (RT_FAILURE(rc))
6991 return rc;
6992
6993 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
6994 pVCpu->hm.s.fLeaveDone = false;
6995 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
6996
6997 return VINF_SUCCESS;
6998}
6999
7000
7001/**
7002 * The thread-context callback (only on platforms which support it).
7003 *
7004 * @param enmEvent The thread-context event.
7005 * @param pVCpu Pointer to the VMCPU.
7006 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7007 * @thread EMT.
7008 */
7009VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7010{
7011 switch (enmEvent)
7012 {
7013 case RTTHREADCTXEVENT_PREEMPTING:
7014 {
7015 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7016 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7017 VMCPU_ASSERT_EMT(pVCpu);
7018
7019 PVM pVM = pVCpu->CTX_SUFF(pVM);
7020 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7021
7022 /* No longjmps (logger flushes, locks) in this fragile context. */
7023 VMMRZCallRing3Disable(pVCpu);
7024 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7025
7026 /*
7027 * Restore host-state (FPU, debug etc.)
7028 */
7029 if (!pVCpu->hm.s.fLeaveDone)
7030 {
7031 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7032 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7033 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7034 pVCpu->hm.s.fLeaveDone = true;
7035 }
7036
7037 /* Leave HM context, takes care of local init (term). */
7038 int rc = HMR0LeaveCpu(pVCpu);
7039 AssertRC(rc); NOREF(rc);
7040
7041 /* Restore longjmp state. */
7042 VMMRZCallRing3Enable(pVCpu);
7043 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7044 break;
7045 }
7046
7047 case RTTHREADCTXEVENT_RESUMED:
7048 {
7049 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7050 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7051 VMCPU_ASSERT_EMT(pVCpu);
7052
7053 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7054 VMMRZCallRing3Disable(pVCpu);
7055 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7056
7057 /* Initialize the bare minimum state required for HM. This takes care of
7058 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7059 int rc = HMR0EnterCpu(pVCpu);
7060 AssertRC(rc);
7061 Assert((pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE))
7062 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7063
7064 /* Load the active VMCS as the current one. */
7065 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7066 {
7067 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7068 AssertRC(rc); NOREF(rc);
7069 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7070 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7071 }
7072 pVCpu->hm.s.fLeaveDone = false;
7073
7074 /* Restore longjmp state. */
7075 VMMRZCallRing3Enable(pVCpu);
7076 break;
7077 }
7078
7079 default:
7080 break;
7081 }
7082}
7083
7084
7085/**
7086 * Saves the host state in the VMCS host-state.
7087 * Sets up the VM-exit MSR-load area.
7088 *
7089 * The CPU state will be loaded from these fields on every successful VM-exit.
7090 *
7091 * @returns VBox status code.
7092 * @param pVM Pointer to the VM.
7093 * @param pVCpu Pointer to the VMCPU.
7094 *
7095 * @remarks No-long-jump zone!!!
7096 */
7097static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7098{
7099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7100
7101 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
7102 return VINF_SUCCESS;
7103
7104 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7105 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7106
7107 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7108 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7109
7110 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7111 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7112
7113 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
7114 return rc;
7115}
7116
7117
7118/**
7119 * Saves the host state in the VMCS host-state.
7120 *
7121 * @returns VBox status code.
7122 * @param pVM Pointer to the VM.
7123 * @param pVCpu Pointer to the VMCPU.
7124 *
7125 * @remarks No-long-jump zone!!!
7126 */
7127VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7128{
7129 AssertPtr(pVM);
7130 AssertPtr(pVCpu);
7131
7132 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7133
7134 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7135 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7136 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7137 return hmR0VmxSaveHostState(pVM, pVCpu);
7138}
7139
7140
7141/**
7142 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7143 * loaded from these fields on every successful VM-entry.
7144 *
7145 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7146 * Sets up the VM-entry controls.
7147 * Sets up the appropriate VMX non-root function to execute guest code based on
7148 * the guest CPU mode.
7149 *
7150 * @returns VBox status code.
7151 * @param pVM Pointer to the VM.
7152 * @param pVCpu Pointer to the VMCPU.
7153 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7154 * out-of-sync. Make sure to update the required fields
7155 * before using them.
7156 *
7157 * @remarks No-long-jump zone!!!
7158 */
7159static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7160{
7161 AssertPtr(pVM);
7162 AssertPtr(pVCpu);
7163 AssertPtr(pMixedCtx);
7164 HMVMX_ASSERT_PREEMPT_SAFE();
7165
7166#ifdef LOG_ENABLED
7167 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7168 * probably not initialized yet? Anyway this will do for now.
7169 *
7170 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7171 * interface and disable ring-3 calls when thread-context hooks are not
7172 * available. */
7173 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7174 VMMR0LogFlushDisable(pVCpu);
7175#endif
7176
7177 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7178
7179 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7180
7181 /* Determine real-on-v86 mode. */
7182 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7183 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7184 && CPUMIsGuestInRealModeEx(pMixedCtx))
7185 {
7186 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7187 }
7188
7189 /*
7190 * Load the guest-state into the VMCS.
7191 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7192 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7193 */
7194 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7195 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7196
7197 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7198 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7199 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7200
7201 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7202 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7203 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7204
7205 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7206 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7207
7208 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7209 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7210
7211 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7212 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7213 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7214
7215 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7216 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7217
7218 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7219 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7220
7221 /*
7222 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7223 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7224 */
7225 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7226 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7227
7228 /* Clear any unused and reserved bits. */
7229 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
7230
7231#ifdef LOG_ENABLED
7232 /* Only reenable log-flushing if the caller has it enabled. */
7233 if (!fCallerDisabledLogFlush)
7234 VMMR0LogFlushEnable(pVCpu);
7235#endif
7236
7237 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7238 return rc;
7239}
7240
7241
7242/**
7243 * Loads the state shared between the host and guest into the VMCS.
7244 *
7245 * @param pVM Pointer to the VM.
7246 * @param pVCpu Pointer to the VMCPU.
7247 * @param pCtx Pointer to the guest-CPU context.
7248 *
7249 * @remarks No-long-jump zone!!!
7250 */
7251static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7252{
7253 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7254 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7255
7256 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
7257 {
7258 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7259 AssertRC(rc);
7260 }
7261
7262 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
7263 {
7264 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7265 AssertRC(rc);
7266
7267 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7268 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
7269 {
7270 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7271 AssertRC(rc);
7272 }
7273 }
7274
7275 AssertMsg(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE), ("fContextUseFlags=%#x\n",
7276 pVCpu->hm.s.fContextUseFlags));
7277}
7278
7279
7280/**
7281 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7282 *
7283 * @param pVM Pointer to the VM.
7284 * @param pVCpu Pointer to the VMCPU.
7285 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7286 * out-of-sync. Make sure to update the required fields
7287 * before using them.
7288 */
7289DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7290{
7291 HMVMX_ASSERT_PREEMPT_SAFE();
7292
7293 Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7294#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7295 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
7296#endif
7297
7298 if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7299 {
7300 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7301 AssertRC(rc);
7302 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7303 }
7304 else if (pVCpu->hm.s.fContextUseFlags)
7305 {
7306 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7307 AssertRC(rc);
7308 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7309 }
7310
7311 /* All the guest state bits should be loaded except maybe the host context and shared host/guest bits. */
7312 AssertMsg( !(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST)
7313 || !(pVCpu->hm.s.fContextUseFlags & ~(HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE)),
7314 ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7315
7316#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7317 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7318 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7319 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7320#endif
7321}
7322
7323
7324/**
7325 * Does the preparations before executing guest code in VT-x.
7326 *
7327 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7328 * recompiler. We must be cautious what we do here regarding committing
7329 * guest-state information into the VMCS assuming we assuredly execute the
7330 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7331 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7332 * so that the recompiler can (and should) use them when it resumes guest
7333 * execution. Otherwise such operations must be done when we can no longer
7334 * exit to ring-3.
7335 *
7336 * @returns Strict VBox status code.
7337 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7338 * have been disabled.
7339 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7340 * double-fault into the guest.
7341 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7342 *
7343 * @param pVM Pointer to the VM.
7344 * @param pVCpu Pointer to the VMCPU.
7345 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7346 * out-of-sync. Make sure to update the required fields
7347 * before using them.
7348 * @param pVmxTransient Pointer to the VMX transient structure.
7349 *
7350 * @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7351 * interrupts will be disabled.
7352 */
7353static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7354{
7355 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7356
7357#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7358 PGMRZDynMapFlushAutoSet(pVCpu);
7359#endif
7360
7361 /* Check force flag actions that might require us to go back to ring-3. */
7362 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7363 if (rc != VINF_SUCCESS)
7364 return rc;
7365
7366#ifndef IEM_VERIFICATION_MODE_FULL
7367 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7368 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7369 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7370 {
7371 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7372 RTGCPHYS GCPhysApicBase;
7373 GCPhysApicBase = pMixedCtx->msrApicBase;
7374 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7375
7376 /* Unalias any existing mapping. */
7377 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7378 AssertRCReturn(rc, rc);
7379
7380 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7381 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7382 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7383 AssertRCReturn(rc, rc);
7384
7385 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7386 }
7387#endif /* !IEM_VERIFICATION_MODE_FULL */
7388
7389 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7390 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7391
7392 /*
7393 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7394 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7395 */
7396 if (TRPMHasTrap(pVCpu))
7397 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7398 else if (!pVCpu->hm.s.Event.fPending)
7399 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7400
7401 /*
7402 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7403 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7404 */
7405 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7406 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7407 {
7408 Assert(rc == VINF_EM_RESET);
7409 return rc;
7410 }
7411
7412 /*
7413 * No longjmps to ring-3 from this point on!!!
7414 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7415 * This also disables flushing of the R0-logger instance (if any).
7416 */
7417 VMMRZCallRing3Disable(pVCpu);
7418
7419 /*
7420 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7421 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7422 *
7423 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7424 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7425 *
7426 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7427 * executing guest code.
7428 */
7429 pVmxTransient->uEflags = ASMIntDisableFlags();
7430 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7431 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7432 {
7433 hmR0VmxClearEventVmcs(pVCpu);
7434 ASMSetFlags(pVmxTransient->uEflags);
7435 VMMRZCallRing3Enable(pVCpu);
7436 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7437 return VINF_EM_RAW_TO_R3;
7438 }
7439 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7440 {
7441 hmR0VmxClearEventVmcs(pVCpu);
7442 ASMSetFlags(pVmxTransient->uEflags);
7443 VMMRZCallRing3Enable(pVCpu);
7444 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7445 return VINF_EM_RAW_INTERRUPT;
7446 }
7447
7448 /* We've injected any pending events. This is really the point of no return (to ring-3). */
7449 pVCpu->hm.s.Event.fPending = false;
7450
7451 return VINF_SUCCESS;
7452}
7453
7454
7455/**
7456 * Prepares to run guest code in VT-x and we've committed to doing so. This
7457 * means there is no backing out to ring-3 or anywhere else at this
7458 * point.
7459 *
7460 * @param pVM Pointer to the VM.
7461 * @param pVCpu Pointer to the VMCPU.
7462 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7463 * out-of-sync. Make sure to update the required fields
7464 * before using them.
7465 * @param pVmxTransient Pointer to the VMX transient structure.
7466 *
7467 * @remarks Called with preemption disabled.
7468 * @remarks No-long-jump zone!!!
7469 */
7470static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7471{
7472 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7473 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7474 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7475
7476 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7477 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7478
7479 /*
7480 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7481 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7482 * Reload only the necessary state, the assertion will catch if other parts of the code
7483 * change.
7484 */
7485 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7486 {
7487 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7488 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7489 }
7490
7491 /*
7492 * Load the host state bits as we may've been preempted (only happens when
7493 * thread-context hooks are used).
7494 */
7495 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
7496 {
7497 /* This ASSUMES that pfnStartVM has been set up already. */
7498 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7499 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7500 AssertRC(rc);
7501 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7502 }
7503 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7504
7505 /*
7506 * Load the state shared between host and guest (FPU, debug).
7507 */
7508 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE)
7509 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7510 AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7511
7512 /*
7513 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7514 */
7515 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7516 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7517
7518 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7519 RTCPUID idCurrentCpu = pCpu->idCpu;
7520 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7521 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
7522 {
7523 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7524 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7525 }
7526
7527 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7528 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7529 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7530 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7531
7532 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7533
7534 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7535 to start executing. */
7536
7537#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7538 /*
7539 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7540 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7541 */
7542 if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7543 && !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7544 {
7545 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7546 uint64_t u64HostTscAux = 0;
7547 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7548 AssertRC(rc2);
7549 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7550 }
7551#endif
7552}
7553
7554
7555/**
7556 * Performs some essential restoration of state after running guest code in
7557 * VT-x.
7558 *
7559 * @param pVM Pointer to the VM.
7560 * @param pVCpu Pointer to the VMCPU.
7561 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7562 * out-of-sync. Make sure to update the required fields
7563 * before using them.
7564 * @param pVmxTransient Pointer to the VMX transient structure.
7565 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7566 *
7567 * @remarks Called with interrupts disabled, and returns with interrups enabled!
7568 *
7569 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7570 * unconditionally when it is safe to do so.
7571 */
7572static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7573{
7574 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7575
7576 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7577 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7578 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7579 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7580 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7581
7582 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7583 {
7584#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7585 /* Restore host's TSC_AUX. */
7586 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7587 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7588#endif
7589 /** @todo Find a way to fix hardcoding a guestimate. */
7590 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7591 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7592 }
7593
7594 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7595 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7596 Assert(!(ASMGetFlags() & X86_EFL_IF));
7597 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7598
7599 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7600 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7601 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7602 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7603
7604 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7605 uint32_t uExitReason;
7606 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7607 rc |= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7608 AssertRC(rc);
7609 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7610 pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7611
7612 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7613 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7614 {
7615 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7616 pVmxTransient->fVMEntryFailed));
7617 return;
7618 }
7619
7620 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7621 {
7622 /* Update the guest interruptibility-state from the VMCS. */
7623 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7624#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7625 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7626 AssertRC(rc);
7627#endif
7628 /*
7629 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7630 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7631 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7632 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7633 */
7634 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7635 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7636 {
7637 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7638 AssertRC(rc);
7639 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
7640 }
7641 }
7642}
7643
7644
7645
7646/**
7647 * Runs the guest code using VT-x the normal way.
7648 *
7649 * @returns VBox status code.
7650 * @param pVM Pointer to the VM.
7651 * @param pVCpu Pointer to the VMCPU.
7652 * @param pCtx Pointer to the guest-CPU context.
7653 *
7654 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
7655 * @remarks Called with preemption disabled.
7656 */
7657static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7658{
7659 VMXTRANSIENT VmxTransient;
7660 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7661 int rc = VERR_INTERNAL_ERROR_5;
7662 uint32_t cLoops = 0;
7663
7664 for (;; cLoops++)
7665 {
7666 Assert(!HMR0SuspendPending());
7667 HMVMX_ASSERT_CPU_SAFE();
7668
7669 /* Preparatory work for running guest code, this may force us to return
7670 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7671 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7672 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7673 if (rc != VINF_SUCCESS)
7674 break;
7675
7676 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7677 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7678 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7679
7680 /* Restore any residual host-state and save any bits shared between host
7681 and guest into the guest-CPU state. Re-enables interrupts! */
7682 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7683
7684 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7685 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7686 {
7687 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7688 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7689 return rc;
7690 }
7691
7692 /* Handle the VM-exit. */
7693 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7694 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7695 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7696 HMVMX_START_EXIT_DISPATCH_PROF();
7697#ifdef HMVMX_USE_FUNCTION_TABLE
7698 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7699#else
7700 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7701#endif
7702 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7703 if (rc != VINF_SUCCESS)
7704 break;
7705 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7706 {
7707 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7708 rc = VINF_EM_RAW_INTERRUPT;
7709 break;
7710 }
7711 }
7712
7713 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7714 return rc;
7715}
7716
7717
7718/**
7719 * Single steps guest code using VT-x.
7720 *
7721 * @returns VBox status code.
7722 * @param pVM Pointer to the VM.
7723 * @param pVCpu Pointer to the VMCPU.
7724 * @param pCtx Pointer to the guest-CPU context.
7725 *
7726 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7727 * @remarks Called with preemption disabled.
7728 */
7729static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7730{
7731 VMXTRANSIENT VmxTransient;
7732 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7733 int rc = VERR_INTERNAL_ERROR_5;
7734 uint32_t cLoops = 0;
7735 uint16_t uCsStart = pCtx->cs.Sel;
7736 uint64_t uRipStart = pCtx->rip;
7737
7738 for (;; cLoops++)
7739 {
7740 Assert(!HMR0SuspendPending());
7741 HMVMX_ASSERT_CPU_SAFE();
7742
7743 /* Preparatory work for running guest code, this may force us to return
7744 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7745 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7746 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7747 if (rc != VINF_SUCCESS)
7748 break;
7749
7750 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7751 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7752 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7753
7754 /* Restore any residual host-state and save any bits shared between host
7755 and guest into the guest-CPU state. Re-enables interrupts! */
7756 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7757
7758 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7759 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7760 {
7761 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7762 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7763 return rc;
7764 }
7765
7766 /* Handle the VM-exit. */
7767 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7768 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7769 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7770 HMVMX_START_EXIT_DISPATCH_PROF();
7771#ifdef HMVMX_USE_FUNCTION_TABLE
7772 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7773#else
7774 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7775#endif
7776 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7777 if (rc != VINF_SUCCESS)
7778 break;
7779 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7780 {
7781 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7782 rc = VINF_EM_RAW_INTERRUPT;
7783 break;
7784 }
7785
7786 /*
7787 * Did the RIP change, if so, consider it a single step.
7788 * Otherwise, make sure one of the TFs gets set.
7789 */
7790 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7791 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7792 AssertRCReturn(rc2, rc2);
7793 if ( pCtx->rip != uRipStart
7794 || pCtx->cs.Sel != uCsStart)
7795 {
7796 rc = VINF_EM_DBG_STEPPED;
7797 break;
7798 }
7799 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
7800 }
7801
7802 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7803 return rc;
7804}
7805
7806
7807/**
7808 * Runs the guest code using VT-x.
7809 *
7810 * @returns VBox status code.
7811 * @param pVM Pointer to the VM.
7812 * @param pVCpu Pointer to the VMCPU.
7813 * @param pCtx Pointer to the guest-CPU context.
7814 *
7815 * @remarks Called with preemption disabled.
7816 */
7817VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7818{
7819 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7820 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
7821 HMVMX_ASSERT_PREEMPT_SAFE();
7822
7823 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
7824
7825 int rc;
7826 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7827 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7828 else
7829 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7830
7831 if (rc == VERR_EM_INTERPRETER)
7832 rc = VINF_EM_RAW_EMULATE_INSTR;
7833 else if (rc == VINF_EM_RESET)
7834 rc = VINF_EM_TRIPLE_FAULT;
7835
7836 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7837 if (RT_FAILURE(rc2))
7838 {
7839 pVCpu->hm.s.u32HMError = rc;
7840 rc = rc2;
7841 }
7842 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
7843 return rc;
7844}
7845
7846
7847#ifndef HMVMX_USE_FUNCTION_TABLE
7848DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7849{
7850 int rc;
7851 switch (rcReason)
7852 {
7853 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7854 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7855 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7856 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7857 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7858 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7859 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7860 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7861 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7862 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7863 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7864 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7865 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7866 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7867 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7868 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7869 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7870 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7871 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7872 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7873 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7874 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7875 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7876 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7877 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7878 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7879 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7880 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7881 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7882 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7883 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7884 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7885 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7886
7887 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7888 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7889 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7890 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7891 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7892 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7893 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7894 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7895 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7896
7897 case VMX_EXIT_VMCALL:
7898 case VMX_EXIT_VMCLEAR:
7899 case VMX_EXIT_VMLAUNCH:
7900 case VMX_EXIT_VMPTRLD:
7901 case VMX_EXIT_VMPTRST:
7902 case VMX_EXIT_VMREAD:
7903 case VMX_EXIT_VMRESUME:
7904 case VMX_EXIT_VMWRITE:
7905 case VMX_EXIT_VMXOFF:
7906 case VMX_EXIT_VMXON:
7907 case VMX_EXIT_INVEPT:
7908 case VMX_EXIT_INVVPID:
7909 case VMX_EXIT_VMFUNC:
7910 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7911 break;
7912 default:
7913 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7914 break;
7915 }
7916 return rc;
7917}
7918#endif
7919
7920#ifdef DEBUG
7921/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7922# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7923 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7924
7925# define HMVMX_ASSERT_PREEMPT_CPUID() \
7926 do \
7927 { \
7928 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7929 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7930 } while (0)
7931
7932# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7933 do { \
7934 AssertPtr(pVCpu); \
7935 AssertPtr(pMixedCtx); \
7936 AssertPtr(pVmxTransient); \
7937 Assert(pVmxTransient->fVMEntryFailed == false); \
7938 Assert(ASMIntAreEnabled()); \
7939 HMVMX_ASSERT_PREEMPT_SAFE(); \
7940 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7941 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7942 HMVMX_ASSERT_PREEMPT_SAFE(); \
7943 if (VMMR0IsLogFlushDisabled(pVCpu)) \
7944 HMVMX_ASSERT_PREEMPT_CPUID(); \
7945 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7946 } while (0)
7947
7948# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7949 do { \
7950 Log4Func(("\n")); \
7951 } while(0)
7952#else /* Release builds */
7953# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7954# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7955#endif
7956
7957
7958/**
7959 * Advances the guest RIP after reading it from the VMCS.
7960 *
7961 * @returns VBox status code.
7962 * @param pVCpu Pointer to the VMCPU.
7963 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7964 * out-of-sync. Make sure to update the required fields
7965 * before using them.
7966 * @param pVmxTransient Pointer to the VMX transient structure.
7967 *
7968 * @remarks No-long-jump zone!!!
7969 */
7970DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7971{
7972 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7973 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7974 AssertRCReturn(rc, rc);
7975
7976 pMixedCtx->rip += pVmxTransient->cbInstr;
7977 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
7978 return rc;
7979}
7980
7981
7982/**
7983 * Tries to determine what part of the guest-state VT-x has deemed as invalid
7984 * and update error record fields accordingly.
7985 *
7986 * @return VMX_IGS_* return codes.
7987 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7988 * wrong with the guest state.
7989 *
7990 * @param pVM Pointer to the VM.
7991 * @param pVCpu Pointer to the VMCPU.
7992 * @param pCtx Pointer to the guest-CPU state.
7993 */
7994static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7995{
7996#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7997#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
7998 uError = (err); \
7999 break; \
8000 } else do {} while (0)
8001/* Duplicate of IEM_IS_CANONICAL(). */
8002#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
8003
8004 int rc;
8005 uint64_t u64Val;
8006 uint32_t u32Val;
8007 uint32_t uError = VMX_IGS_ERROR;
8008 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8009
8010 do
8011 {
8012 /*
8013 * CR0.
8014 */
8015 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8016 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8017 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8018 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8019 if (fUnrestrictedGuest)
8020 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8021
8022 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
8023 AssertRCBreak(rc);
8024 HMVMX_CHECK_BREAK((u32Val & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8025 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8026 if ( !fUnrestrictedGuest
8027 && (u32Val & X86_CR0_PG)
8028 && !(u32Val & X86_CR0_PE))
8029 {
8030 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8031 }
8032
8033 /*
8034 * CR4.
8035 */
8036 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8037 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8038 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
8039 AssertRCBreak(rc);
8040 HMVMX_CHECK_BREAK((u32Val & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8041 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8042
8043 /*
8044 * IA32_DEBUGCTL MSR.
8045 */
8046 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8047 AssertRCBreak(rc);
8048 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8049 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8050 {
8051 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8052 }
8053 uint64_t u64DebugCtlMsr = u64Val;
8054
8055#ifdef VBOX_STRICT
8056 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
8057 AssertRCBreak(rc);
8058 Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8059#endif
8060 bool const fLongModeGuest = !!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8061
8062 /*
8063 * RIP and RFLAGS.
8064 */
8065 uint32_t u32Eflags;
8066#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8067 if (HMVMX_IS_64BIT_HOST_MODE())
8068 {
8069 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8070 AssertRCBreak(rc);
8071 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8072 if ( !fLongModeGuest
8073 || !pCtx->cs.Attr.n.u1Long)
8074 {
8075 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8076 }
8077 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8078 * must be identical if the "IA32e mode guest" VM-entry control is 1
8079 * and CS.L is 1. No check applies if the CPU supports 64
8080 * linear-address bits. */
8081
8082 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8083 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8084 AssertRCBreak(rc);
8085 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8086 VMX_IGS_RFLAGS_RESERVED);
8087 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8088 u32Eflags = u64Val;
8089 }
8090 else
8091#endif
8092 {
8093 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8094 AssertRCBreak(rc);
8095 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8096 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8097 }
8098
8099 if ( fLongModeGuest
8100 || !(pCtx->cr0 & X86_CR0_PE))
8101 {
8102 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8103 }
8104
8105 uint32_t u32EntryInfo;
8106 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8107 AssertRCBreak(rc);
8108 if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8109 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8110 {
8111 HMVMX_CHECK_BREAK(u32Val & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8112 }
8113
8114 /*
8115 * 64-bit checks.
8116 */
8117#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8118 if (HMVMX_IS_64BIT_HOST_MODE())
8119 {
8120 if ( fLongModeGuest
8121 && !fUnrestrictedGuest)
8122 {
8123 HMVMX_CHECK_BREAK(CPUMIsGuestPagingEnabledEx(pCtx), VMX_IGS_CR0_PG_LONGMODE);
8124 HMVMX_CHECK_BREAK((pCtx->cr4 & X86_CR4_PAE), VMX_IGS_CR4_PAE_LONGMODE);
8125 }
8126
8127 if ( !fLongModeGuest
8128 && (pCtx->cr4 & X86_CR4_PCIDE))
8129 {
8130 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8131 }
8132
8133 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8134 * 51:32 beyond the processor's physical-address width are 0. */
8135
8136 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8137 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8138 {
8139 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8140 }
8141
8142 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8143 AssertRCBreak(rc);
8144 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8145
8146 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8147 AssertRCBreak(rc);
8148 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8149 }
8150#endif
8151
8152 /*
8153 * PERF_GLOBAL MSR.
8154 */
8155 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8156 {
8157 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8158 AssertRCBreak(rc);
8159 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8160 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8161 }
8162
8163 /*
8164 * PAT MSR.
8165 */
8166 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8167 {
8168 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8169 AssertRCBreak(rc);
8170 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8171 for (unsigned i = 0; i < 8; i++)
8172 {
8173 uint8_t u8Val = (u64Val & 0x7);
8174 if ( u8Val != 0 /* UC */
8175 || u8Val != 1 /* WC */
8176 || u8Val != 4 /* WT */
8177 || u8Val != 5 /* WP */
8178 || u8Val != 6 /* WB */
8179 || u8Val != 7 /* UC- */)
8180 {
8181 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8182 }
8183 u64Val >>= 3;
8184 }
8185 }
8186
8187 /*
8188 * EFER MSR.
8189 */
8190 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8191 {
8192 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8193 AssertRCBreak(rc);
8194 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8195 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8196 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8197 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8198 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8199 || (u64Val & MSR_K6_EFER_LMA) == (pCtx->cr0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8200 }
8201
8202 /*
8203 * Segment registers.
8204 */
8205 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8206 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8207 if (!(u32Eflags & X86_EFL_VM))
8208 {
8209 /* CS */
8210 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8211 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8212 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8213 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8214 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8215 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8216 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8217 /* CS cannot be loaded with NULL in protected mode. */
8218 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8219 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8220 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8221 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8222 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8223 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8224 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8225 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8226 else
8227 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8228
8229 /* SS */
8230 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8231 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8232 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8233 if ( !(pCtx->cr0 & X86_CR0_PE)
8234 || pCtx->cs.Attr.n.u4Type == 3)
8235 {
8236 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8237 }
8238 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8239 {
8240 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8241 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8242 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8243 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8244 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8245 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8246 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8247 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8248 }
8249
8250 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8251 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8252 {
8253 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8254 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8255 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8256 || pCtx->ds.Attr.n.u4Type > 11
8257 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8258 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8259 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8260 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8261 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8262 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8263 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8264 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8265 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8266 }
8267 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8268 {
8269 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8270 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8271 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8272 || pCtx->es.Attr.n.u4Type > 11
8273 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8274 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8275 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8276 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8277 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8278 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8279 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8280 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8281 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8282 }
8283 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8284 {
8285 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8286 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8287 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8288 || pCtx->fs.Attr.n.u4Type > 11
8289 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8290 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8291 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8292 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8293 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8294 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8295 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8296 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8297 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8298 }
8299 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8300 {
8301 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8302 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8303 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8304 || pCtx->gs.Attr.n.u4Type > 11
8305 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8306 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8307 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8308 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8309 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8310 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8311 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8312 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8313 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8314 }
8315 /* 64-bit capable CPUs. */
8316#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8317 if (HMVMX_IS_64BIT_HOST_MODE())
8318 {
8319 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8320 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8321 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8322 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8323 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8324 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8325 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8326 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8327 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8328 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8329 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8330 }
8331#endif
8332 }
8333 else
8334 {
8335 /* V86 mode checks. */
8336 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8337 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8338 {
8339 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8340 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8341 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8342 }
8343 else
8344 {
8345 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8346 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8347 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8348 }
8349
8350 /* CS */
8351 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8352 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8353 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8354 /* SS */
8355 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8356 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8357 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8358 /* DS */
8359 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8360 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8361 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8362 /* ES */
8363 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8364 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8365 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8366 /* FS */
8367 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8368 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8369 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8370 /* GS */
8371 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8372 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8373 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8374 /* 64-bit capable CPUs. */
8375#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8376 if (HMVMX_IS_64BIT_HOST_MODE())
8377 {
8378 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8379 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8380 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8381 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8382 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8383 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8384 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8385 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8386 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8387 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8388 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8389 }
8390#endif
8391 }
8392
8393 /*
8394 * TR.
8395 */
8396 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8397 /* 64-bit capable CPUs. */
8398#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8399 if (HMVMX_IS_64BIT_HOST_MODE())
8400 {
8401 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8402 }
8403#endif
8404 if (fLongModeGuest)
8405 {
8406 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8407 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8408 }
8409 else
8410 {
8411 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8412 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8413 VMX_IGS_TR_ATTR_TYPE_INVALID);
8414 }
8415 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8416 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8417 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8418 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8419 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8420 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8421 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8422 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8423
8424 /*
8425 * GDTR and IDTR.
8426 */
8427#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8428 if (HMVMX_IS_64BIT_HOST_MODE())
8429 {
8430 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8431 AssertRCBreak(rc);
8432 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8433
8434 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8435 AssertRCBreak(rc);
8436 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8437 }
8438#endif
8439
8440 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8441 AssertRCBreak(rc);
8442 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8443
8444 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8445 AssertRCBreak(rc);
8446 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8447
8448 /*
8449 * Guest Non-Register State.
8450 */
8451 /* Activity State. */
8452 uint32_t u32ActivityState;
8453 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8454 AssertRCBreak(rc);
8455 HMVMX_CHECK_BREAK( !u32ActivityState
8456 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8457 VMX_IGS_ACTIVITY_STATE_INVALID);
8458 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8459 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8460 uint32_t u32IntrState;
8461 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8462 AssertRCBreak(rc);
8463 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8464 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8465 {
8466 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8467 }
8468
8469 /** @todo Activity state and injecting interrupts. Left as a todo since we
8470 * currently don't use activity states but ACTIVE. */
8471
8472 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8473 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8474
8475 /* Guest interruptibility-state. */
8476 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8477 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8478 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8479 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8480 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8481 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8482 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8483 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8484 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8485 if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8486 {
8487 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8488 {
8489 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8490 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8491 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8492 }
8493 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8494 {
8495 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8496 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8497 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8498 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8499 }
8500 }
8501 /** @todo Assumes the processor is not in SMM. */
8502 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8503 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8504 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8505 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8506 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8507 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8508 && VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8509 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8510 {
8511 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8512 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8513 }
8514
8515 /* Pending debug exceptions. */
8516 if (HMVMX_IS_64BIT_HOST_MODE())
8517 {
8518 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8519 AssertRCBreak(rc);
8520 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8521 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8522 u32Val = u64Val; /* For pending debug exceptions checks below. */
8523 }
8524 else
8525 {
8526 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8527 AssertRCBreak(rc);
8528 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8529 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8530 }
8531
8532 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8533 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8534 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8535 {
8536 if ( (u32Eflags & X86_EFL_TF)
8537 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8538 {
8539 /* Bit 14 is PendingDebug.BS. */
8540 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8541 }
8542 if ( !(u32Eflags & X86_EFL_TF)
8543 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8544 {
8545 /* Bit 14 is PendingDebug.BS. */
8546 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8547 }
8548 }
8549
8550 /* VMCS link pointer. */
8551 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8552 AssertRCBreak(rc);
8553 if (u64Val != UINT64_C(0xffffffffffffffff))
8554 {
8555 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8556 /** @todo Bits beyond the processor's physical-address width MBZ. */
8557 /** @todo 32-bit located in memory referenced by value of this field (as a
8558 * physical address) must contain the processor's VMCS revision ID. */
8559 /** @todo SMM checks. */
8560 }
8561
8562 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8563
8564 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8565 if (uError == VMX_IGS_ERROR)
8566 uError = VMX_IGS_REASON_NOT_FOUND;
8567 } while (0);
8568
8569 pVCpu->hm.s.u32HMError = uError;
8570 return uError;
8571
8572#undef HMVMX_ERROR_BREAK
8573#undef HMVMX_CHECK_BREAK
8574#undef HMVMX_IS_CANONICAL
8575}
8576
8577/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8578/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8579/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8580
8581/** @name VM-exit handlers.
8582 * @{
8583 */
8584
8585/**
8586 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8587 */
8588HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8589{
8590 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8591 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8592 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8593#if HC_ARCH_BITS == 64
8594 Assert(ASMIntAreEnabled());
8595 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8596 return VINF_SUCCESS;
8597#endif
8598 return VINF_EM_RAW_INTERRUPT;
8599}
8600
8601
8602/**
8603 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8604 */
8605HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8606{
8607 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8608 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8609
8610 int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8611 AssertRCReturn(rc, rc);
8612
8613 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8614 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8615 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8616 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8617
8618 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8619 {
8620 /*
8621 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8622 * anything we inject is not going to cause a VM-exit directly for the event being injected.
8623 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8624 *
8625 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8626 */
8627 VMXDispatchHostNmi();
8628 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
8629 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8630 return VINF_SUCCESS;
8631 }
8632
8633 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8634 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8635 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8636 {
8637 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8638 return VINF_SUCCESS;
8639 }
8640 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8641 {
8642 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8643 return rc;
8644 }
8645
8646 uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8647 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8648 switch (uIntrType)
8649 {
8650 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8651 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8652 /* no break */
8653 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8654 {
8655 switch (uVector)
8656 {
8657 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8658 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8659 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8660 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8661 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8662 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8663#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8664 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8665 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8666 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8667 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8668 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8669 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8670 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8671 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8672 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8673 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8674#endif
8675 default:
8676 {
8677 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8678 AssertRCReturn(rc, rc);
8679
8680 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8681 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8682 {
8683 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8684 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8685 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8686
8687 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8688 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8689 AssertRCReturn(rc, rc);
8690 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8691 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8692 0 /* GCPtrFaultAddress */);
8693 AssertRCReturn(rc, rc);
8694 }
8695 else
8696 {
8697 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8698 pVCpu->hm.s.u32HMError = uVector;
8699 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8700 }
8701 break;
8702 }
8703 }
8704 break;
8705 }
8706
8707 default:
8708 {
8709 pVCpu->hm.s.u32HMError = uExitIntrInfo;
8710 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8711 AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8712 break;
8713 }
8714 }
8715 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8716 return rc;
8717}
8718
8719
8720/**
8721 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8722 */
8723HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8724{
8725 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8726
8727 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8728 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8729 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8730 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8731 AssertRCReturn(rc, rc);
8732
8733 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8734 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8735 return VINF_SUCCESS;
8736}
8737
8738
8739/**
8740 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8741 */
8742HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8743{
8744 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8745 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8746 pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8747 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8748}
8749
8750
8751/**
8752 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8753 */
8754HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8755{
8756 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8757 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8758 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8759}
8760
8761
8762/**
8763 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8764 */
8765HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8766{
8767 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8768 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8769 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8770}
8771
8772
8773/**
8774 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8775 */
8776HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8777{
8778 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8779 PVM pVM = pVCpu->CTX_SUFF(pVM);
8780 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8781 if (RT_LIKELY(rc == VINF_SUCCESS))
8782 {
8783 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8784 Assert(pVmxTransient->cbInstr == 2);
8785 }
8786 else
8787 {
8788 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8789 rc = VERR_EM_INTERPRETER;
8790 }
8791 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8792 return rc;
8793}
8794
8795
8796/**
8797 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8798 */
8799HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8800{
8801 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8802 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8803 AssertRCReturn(rc, rc);
8804
8805 if (pMixedCtx->cr4 & X86_CR4_SMXE)
8806 return VINF_EM_RAW_EMULATE_INSTR;
8807
8808 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8809 pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8810 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8811}
8812
8813
8814/**
8815 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8816 */
8817HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8818{
8819 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8820 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8821 AssertRCReturn(rc, rc);
8822
8823 PVM pVM = pVCpu->CTX_SUFF(pVM);
8824 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8825 if (RT_LIKELY(rc == VINF_SUCCESS))
8826 {
8827 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8828 Assert(pVmxTransient->cbInstr == 2);
8829 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8830 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8831 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8832 }
8833 else
8834 {
8835 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8836 rc = VERR_EM_INTERPRETER;
8837 }
8838 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8839 return rc;
8840}
8841
8842
8843/**
8844 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8845 */
8846HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8847{
8848 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8849 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8850 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8851 AssertRCReturn(rc, rc);
8852
8853 PVM pVM = pVCpu->CTX_SUFF(pVM);
8854 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8855 if (RT_LIKELY(rc == VINF_SUCCESS))
8856 {
8857 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8858 Assert(pVmxTransient->cbInstr == 3);
8859 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8860 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8861 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8862 }
8863 else
8864 {
8865 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8866 rc = VERR_EM_INTERPRETER;
8867 }
8868 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8869 return rc;
8870}
8871
8872
8873/**
8874 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8875 */
8876HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8877{
8878 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8879 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8880 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8881 AssertRCReturn(rc, rc);
8882
8883 PVM pVM = pVCpu->CTX_SUFF(pVM);
8884 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8885 if (RT_LIKELY(rc == VINF_SUCCESS))
8886 {
8887 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8888 Assert(pVmxTransient->cbInstr == 2);
8889 }
8890 else
8891 {
8892 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8893 rc = VERR_EM_INTERPRETER;
8894 }
8895 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8896 return rc;
8897}
8898
8899
8900/**
8901 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8902 */
8903HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8904{
8905 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8906 PVM pVM = pVCpu->CTX_SUFF(pVM);
8907 Assert(!pVM->hm.s.fNestedPaging);
8908
8909 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8910 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8911 AssertRCReturn(rc, rc);
8912
8913 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8914 rc = VBOXSTRICTRC_VAL(rc2);
8915 if (RT_LIKELY(rc == VINF_SUCCESS))
8916 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8917 else
8918 {
8919 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8920 pVmxTransient->uExitQualification, rc));
8921 }
8922 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8923 return rc;
8924}
8925
8926
8927/**
8928 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8929 */
8930HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8931{
8932 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8933 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8934 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8935 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8936 AssertRCReturn(rc, rc);
8937
8938 PVM pVM = pVCpu->CTX_SUFF(pVM);
8939 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8940 if (RT_LIKELY(rc == VINF_SUCCESS))
8941 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8942 else
8943 {
8944 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8945 rc = VERR_EM_INTERPRETER;
8946 }
8947 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8948 return rc;
8949}
8950
8951
8952/**
8953 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8954 */
8955HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8956{
8957 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8958 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8959 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8960 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8961 AssertRCReturn(rc, rc);
8962
8963 PVM pVM = pVCpu->CTX_SUFF(pVM);
8964 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8965 rc = VBOXSTRICTRC_VAL(rc2);
8966 if (RT_LIKELY( rc == VINF_SUCCESS
8967 || rc == VINF_EM_HALT))
8968 {
8969 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8970 AssertRCReturn(rc3, rc3);
8971
8972 if ( rc == VINF_EM_HALT
8973 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
8974 {
8975 rc = VINF_SUCCESS;
8976 }
8977 }
8978 else
8979 {
8980 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
8981 rc = VERR_EM_INTERPRETER;
8982 }
8983 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
8984 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
8985 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
8986 return rc;
8987}
8988
8989
8990/**
8991 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
8992 */
8993HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8994{
8995 /*
8996 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
8997 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
8998 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
8999 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9000 */
9001 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9002 pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
9003 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9004}
9005
9006
9007/**
9008 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9009 */
9010HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9011{
9012 /*
9013 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9014 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9015 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9016 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9017 */
9018 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9019 pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
9020 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9021}
9022
9023
9024/**
9025 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9026 */
9027HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9028{
9029 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9030 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9031 pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
9032 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9033}
9034
9035
9036/**
9037 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9038 */
9039HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9040{
9041 /*
9042 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9043 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9044 * See Intel spec. 25.3 "Other Causes of VM-exits".
9045 */
9046 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9047 pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
9048 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9049}
9050
9051
9052/**
9053 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9054 * VM-exit.
9055 */
9056HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9057{
9058 /*
9059 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9060 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9061 *
9062 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9063 * See Intel spec. "23.8 Restrictions on VMX operation".
9064 */
9065 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9066 return VINF_SUCCESS;
9067}
9068
9069
9070/**
9071 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9072 * VM-exit.
9073 */
9074HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9075{
9076 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9077 return VINF_EM_RESET;
9078}
9079
9080
9081/**
9082 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9083 */
9084HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9085{
9086 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9087 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9088 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9089 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9090 AssertRCReturn(rc, rc);
9091
9092 pMixedCtx->rip++;
9093 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9094 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9095 rc = VINF_SUCCESS;
9096 else
9097 rc = VINF_EM_HALT;
9098
9099 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9100 return rc;
9101}
9102
9103
9104/**
9105 * VM-exit handler for instructions that result in a #UD exception delivered to
9106 * the guest.
9107 */
9108HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9109{
9110 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9111 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9112 return VINF_SUCCESS;
9113}
9114
9115
9116/**
9117 * VM-exit handler for expiry of the VMX preemption timer.
9118 */
9119HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9120{
9121 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9122
9123 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9124 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9125
9126 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9127 PVM pVM = pVCpu->CTX_SUFF(pVM);
9128 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9129 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9130 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9131}
9132
9133
9134/**
9135 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9136 */
9137HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9138{
9139 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9140
9141 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9142 /** @todo check if XSETBV is supported by the recompiler. */
9143 return VERR_EM_INTERPRETER;
9144}
9145
9146
9147/**
9148 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9149 */
9150HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9151{
9152 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9153
9154 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9155 /** @todo implement EMInterpretInvpcid() */
9156 return VERR_EM_INTERPRETER;
9157}
9158
9159
9160/**
9161 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9162 * Error VM-exit.
9163 */
9164HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9165{
9166 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9167 AssertRCReturn(rc, rc);
9168
9169 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9170 NOREF(uInvalidReason);
9171
9172#ifdef VBOX_STRICT
9173 uint32_t uIntrState;
9174 HMVMXHCUINTREG uHCReg;
9175 uint64_t u64Val;
9176 uint32_t u32Val;
9177
9178 rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
9179 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9180 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9181 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9182 AssertRCReturn(rc, rc);
9183
9184 Log4(("uInvalidReason %u\n", uInvalidReason));
9185 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
9186 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9187 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9188 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9189
9190 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9191 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9192 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9193 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9194 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9195 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9196 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9197 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9198 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9199 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9200 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9201 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9202#endif
9203
9204 PVM pVM = pVCpu->CTX_SUFF(pVM);
9205 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9206
9207 return VERR_VMX_INVALID_GUEST_STATE;
9208}
9209
9210
9211/**
9212 * VM-exit handler for VM-entry failure due to an MSR-load
9213 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9214 */
9215HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9216{
9217 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9218 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9219}
9220
9221
9222/**
9223 * VM-exit handler for VM-entry failure due to a machine-check event
9224 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9225 */
9226HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9227{
9228 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9229 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9230}
9231
9232
9233/**
9234 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9235 * theory.
9236 */
9237HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9238{
9239 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9240 return VERR_VMX_UNDEFINED_EXIT_CODE;
9241}
9242
9243
9244/**
9245 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9246 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9247 * Conditional VM-exit.
9248 */
9249HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9250{
9251 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9252
9253 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9254 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9255 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9256 return VERR_EM_INTERPRETER;
9257 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9258 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9259}
9260
9261
9262/**
9263 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9264 */
9265HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9266{
9267 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9268
9269 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9270 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9271 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9272 return VERR_EM_INTERPRETER;
9273 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9274 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9275}
9276
9277
9278/**
9279 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9280 */
9281HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9282{
9283 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9284
9285 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9286 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9287 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9288 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9289 AssertRCReturn(rc, rc);
9290 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9291
9292 PVM pVM = pVCpu->CTX_SUFF(pVM);
9293 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9294 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9295 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9296 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9297
9298 if (RT_LIKELY(rc == VINF_SUCCESS))
9299 {
9300 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9301 Assert(pVmxTransient->cbInstr == 2);
9302 }
9303 return rc;
9304}
9305
9306
9307/**
9308 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9309 */
9310HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9311{
9312 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9313 PVM pVM = pVCpu->CTX_SUFF(pVM);
9314 int rc = VINF_SUCCESS;
9315
9316 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9317 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9318 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9319 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9320 AssertRCReturn(rc, rc);
9321 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9322
9323 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9324 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9325 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9326
9327 if (RT_LIKELY(rc == VINF_SUCCESS))
9328 {
9329 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9330
9331 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9332 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9333 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9334 {
9335 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9336 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9337 EMInterpretWrmsr() changes it. */
9338 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9339 }
9340 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9341 {
9342 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9343 AssertRCReturn(rc, rc);
9344 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
9345 }
9346 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9347 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9348
9349 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9350 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9351 {
9352 switch (pMixedCtx->ecx)
9353 {
9354 case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
9355 case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
9356 case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
9357 case MSR_K8_FS_BASE: /* no break */
9358 case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS; break;
9359 case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
9360 }
9361 }
9362#ifdef VBOX_STRICT
9363 else
9364 {
9365 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9366 switch (pMixedCtx->ecx)
9367 {
9368 case MSR_IA32_SYSENTER_CS:
9369 case MSR_IA32_SYSENTER_EIP:
9370 case MSR_IA32_SYSENTER_ESP:
9371 case MSR_K8_FS_BASE:
9372 case MSR_K8_GS_BASE:
9373 {
9374 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9375 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9376 }
9377
9378 case MSR_K8_LSTAR:
9379 case MSR_K6_STAR:
9380 case MSR_K8_SF_MASK:
9381 case MSR_K8_TSC_AUX:
9382 case MSR_K8_KERNEL_GS_BASE:
9383 {
9384 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9385 pMixedCtx->ecx));
9386 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9387 }
9388 }
9389 }
9390#endif /* VBOX_STRICT */
9391 }
9392 return rc;
9393}
9394
9395
9396/**
9397 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9398 */
9399HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9400{
9401 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9402
9403 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9404 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9405 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9406 return VERR_EM_INTERPRETER;
9407 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9408 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9409}
9410
9411
9412/**
9413 * VM-exit handler for when the TPR value is lowered below the specified
9414 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9415 */
9416HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9417{
9418 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9419 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9420
9421 /*
9422 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9423 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9424 * resume guest execution.
9425 */
9426 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9427 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9428 return VINF_SUCCESS;
9429}
9430
9431
9432/**
9433 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9434 * VM-exit.
9435 *
9436 * @retval VINF_SUCCESS when guest execution can continue.
9437 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9438 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9439 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9440 * recompiler.
9441 */
9442HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9443{
9444 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9445 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9446 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9447 AssertRCReturn(rc, rc);
9448
9449 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9450 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9451 PVM pVM = pVCpu->CTX_SUFF(pVM);
9452 switch (uAccessType)
9453 {
9454 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9455 {
9456#if 0
9457 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9458 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9459#else
9460 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9461 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9462 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9463#endif
9464 AssertRCReturn(rc, rc);
9465
9466 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9467 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9468 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9469 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9470
9471 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9472 {
9473 case 0: /* CR0 */
9474 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9475 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9476 break;
9477 case 2: /* C2 **/
9478 /* Nothing to do here, CR2 it's not part of the VMCS. */
9479 break;
9480 case 3: /* CR3 */
9481 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9482 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9483 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR3;
9484 break;
9485 case 4: /* CR4 */
9486 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9487 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR4;
9488 break;
9489 case 8: /* CR8 */
9490 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9491 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9492 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9493 break;
9494 default:
9495 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9496 break;
9497 }
9498
9499 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9500 break;
9501 }
9502
9503 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9504 {
9505 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9506 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9507 AssertRCReturn(rc, rc);
9508 Assert( !pVM->hm.s.fNestedPaging
9509 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9510 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9511
9512 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9513 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9514 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9515
9516 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9517 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9518 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9519 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9520 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9521 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9522 break;
9523 }
9524
9525 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9526 {
9527 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9528 AssertRCReturn(rc, rc);
9529 rc = EMInterpretCLTS(pVM, pVCpu);
9530 AssertRCReturn(rc, rc);
9531 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9532 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9533 Log4(("CRX CLTS write rc=%d\n", rc));
9534 break;
9535 }
9536
9537 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9538 {
9539 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9540 AssertRCReturn(rc, rc);
9541 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9542 if (RT_LIKELY(rc == VINF_SUCCESS))
9543 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9544 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9545 Log4(("CRX LMSW write rc=%d\n", rc));
9546 break;
9547 }
9548
9549 default:
9550 {
9551 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9552 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9553 }
9554 }
9555
9556 /* Validate possible error codes. */
9557 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9558 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9559 if (RT_SUCCESS(rc))
9560 {
9561 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9562 AssertRCReturn(rc2, rc2);
9563 }
9564
9565 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9566 return rc;
9567}
9568
9569
9570/**
9571 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9572 * VM-exit.
9573 */
9574HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9575{
9576 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9577 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9578
9579 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9580 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9581 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9582 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9583 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9584 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9585 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9586 AssertRCReturn(rc2, rc2);
9587
9588 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9589 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9590 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9591 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9592 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9593 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9594 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9595
9596 /* I/O operation lookup arrays. */
9597 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9598 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9599
9600 VBOXSTRICTRC rcStrict;
9601 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9602 const uint32_t cbInstr = pVmxTransient->cbInstr;
9603 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9604 PVM pVM = pVCpu->CTX_SUFF(pVM);
9605 if (fIOString)
9606 {
9607 /*
9608 * INS/OUTS - I/O String instruction.
9609 *
9610 * Use instruction-information if available, otherwise fall back on
9611 * interpreting the instruction.
9612 */
9613 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9614#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9615 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9616 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9617 {
9618 rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9619 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9620 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9621 AssertRCReturn(rc2, rc2);
9622 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9623 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9624 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9625 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9626 if (fIOWrite)
9627 {
9628 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9629 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9630 //if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9631 // hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9632 // pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9633 }
9634 else
9635 {
9636 AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9637 ("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9638 VERR_HMVMX_IPE_4);
9639 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9640 //if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9641 // hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9642 }
9643 }
9644 else
9645 {
9646 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9647 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9648 AssertRCReturn(rc2, rc2);
9649 rcStrict = IEMExecOne(pVCpu);
9650 }
9651 /** @todo IEM needs to be setting these flags somehow. */
9652 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9653 fUpdateRipAlready = true;
9654#else
9655 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9656 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9657 if (RT_SUCCESS(rcStrict))
9658 {
9659 if (fIOWrite)
9660 {
9661 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9662 (DISCPUMODE)pDis->uAddrMode, cbValue);
9663 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9664 }
9665 else
9666 {
9667 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9668 (DISCPUMODE)pDis->uAddrMode, cbValue);
9669 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9670 }
9671 }
9672 else
9673 {
9674 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9675 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9676 }
9677#endif
9678 }
9679 else
9680 {
9681 /*
9682 * IN/OUT - I/O instruction.
9683 */
9684 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9685 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9686 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9687 if (fIOWrite)
9688 {
9689 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9690 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9691 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9692 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9693 }
9694 else
9695 {
9696 uint32_t u32Result = 0;
9697 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9698 if (IOM_SUCCESS(rcStrict))
9699 {
9700 /* Save result of I/O IN instr. in AL/AX/EAX. */
9701 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
9702 }
9703 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9704 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9705 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9706 }
9707 }
9708
9709 if (IOM_SUCCESS(rcStrict))
9710 {
9711 if (!fUpdateRipAlready)
9712 {
9713 pMixedCtx->rip += cbInstr;
9714 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9715 }
9716
9717 /*
9718 * If any I/O breakpoints are armed, we need to check if one triggered
9719 * and take appropriate action.
9720 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9721 */
9722 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9723 AssertRCReturn(rc2, rc2);
9724
9725 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9726 * execution engines about whether hyper BPs and such are pending. */
9727 uint32_t const uDr7 = pMixedCtx->dr[7];
9728 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9729 && X86_DR7_ANY_RW_IO(uDr7)
9730 && (pMixedCtx->cr4 & X86_CR4_DE))
9731 || DBGFBpIsHwIoArmed(pVM)))
9732 {
9733 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9734
9735 /* We're playing with the host CPU state here, make sure we don't preempt. */
9736 HM_DISABLE_PREEMPT_IF_NEEDED();
9737 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
9738
9739 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9740 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9741 {
9742 /* Raise #DB. */
9743 if (fIsGuestDbgActive)
9744 ASMSetDR6(pMixedCtx->dr[6]);
9745 if (pMixedCtx->dr[7] != uDr7)
9746 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9747
9748 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9749 }
9750 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9751 else if ( rcStrict2 != VINF_SUCCESS
9752 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
9753 rcStrict = rcStrict2;
9754
9755 HM_RESTORE_PREEMPT_IF_NEEDED();
9756 }
9757 }
9758
9759#ifdef DEBUG
9760 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9761 Assert(!fIOWrite);
9762 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9763 Assert(fIOWrite);
9764 else
9765 {
9766 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9767 * statuses, that the VMM device and some others may return. See
9768 * IOM_SUCCESS() for guidance. */
9769 AssertMsg( RT_FAILURE(rcStrict)
9770 || rcStrict == VINF_SUCCESS
9771 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
9772 || rcStrict == VINF_EM_DBG_BREAKPOINT
9773 || rcStrict == VINF_EM_RAW_GUEST_TRAP
9774 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9775 }
9776#endif
9777
9778 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9779 return VBOXSTRICTRC_TODO(rcStrict);
9780}
9781
9782
9783/**
9784 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9785 * VM-exit.
9786 */
9787HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9788{
9789 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9790
9791 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
9792 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9793 AssertRCReturn(rc, rc);
9794 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9795 {
9796 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9797 AssertRCReturn(rc, rc);
9798 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9799 {
9800 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9801
9802 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9803 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9804 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9805 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9806 {
9807 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9808 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9809
9810 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9811 Assert(!pVCpu->hm.s.Event.fPending);
9812 pVCpu->hm.s.Event.fPending = true;
9813 pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9814 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9815 AssertRCReturn(rc, rc);
9816 if (fErrorCodeValid)
9817 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9818 else
9819 pVCpu->hm.s.Event.u32ErrCode = 0;
9820 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9821 && uVector == X86_XCPT_PF)
9822 {
9823 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9824 }
9825
9826 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9827 }
9828 }
9829 }
9830
9831 /** @todo Emulate task switch someday, currently just going back to ring-3 for
9832 * emulation. */
9833 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9834 return VERR_EM_INTERPRETER;
9835}
9836
9837
9838/**
9839 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9840 */
9841HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9842{
9843 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9844 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9845 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9846 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9847 AssertRCReturn(rc, rc);
9848 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9849 return VINF_EM_DBG_STEPPED;
9850}
9851
9852
9853/**
9854 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9855 */
9856HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9857{
9858 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9859
9860 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9861 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9862 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9863 return VINF_SUCCESS;
9864 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9865 return rc;
9866
9867#if 0
9868 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9869 * just sync the whole thing. */
9870 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9871#else
9872 /* Aggressive state sync. for now. */
9873 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9874 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9875 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9876#endif
9877 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9878 AssertRCReturn(rc, rc);
9879
9880 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9881 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9882 switch (uAccessType)
9883 {
9884 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9885 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9886 {
9887 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9888 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9889 {
9890 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9891 }
9892
9893 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9894 GCPhys &= PAGE_BASE_GC_MASK;
9895 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9896 PVM pVM = pVCpu->CTX_SUFF(pVM);
9897 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9898 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9899
9900 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9901 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9902 CPUMCTX2CORE(pMixedCtx), GCPhys);
9903 rc = VBOXSTRICTRC_VAL(rc2);
9904 Log4(("ApicAccess rc=%d\n", rc));
9905 if ( rc == VINF_SUCCESS
9906 || rc == VERR_PAGE_TABLE_NOT_PRESENT
9907 || rc == VERR_PAGE_NOT_PRESENT)
9908 {
9909 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
9910 | HM_CHANGED_VMX_GUEST_APIC_STATE;
9911 rc = VINF_SUCCESS;
9912 }
9913 break;
9914 }
9915
9916 default:
9917 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9918 rc = VINF_EM_RAW_EMULATE_INSTR;
9919 break;
9920 }
9921
9922 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9923 return rc;
9924}
9925
9926
9927/**
9928 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9929 * VM-exit.
9930 */
9931HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9932{
9933 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9934
9935 /* We should -not- get this VM-exit if the guest's debug registers are active. See CPUMR0LoadGuestDebugState(). */
9936#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9937 if ( !CPUMIsGuestInLongModeEx(pMixedCtx) /* EFER is always up-to-date. */
9938 && CPUMIsGuestDebugStateActive(pVCpu))
9939#else
9940 if (CPUMIsGuestDebugStateActive(pVCpu))
9941#endif
9942 {
9943 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9944 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9945 }
9946
9947 int rc = VERR_INTERNAL_ERROR_5;
9948 if ( !DBGFIsStepping(pVCpu)
9949 && !pVCpu->hm.s.fSingleInstruction
9950 && !CPUMIsHyperDebugStateActive(pVCpu))
9951 {
9952 /* Don't intercept MOV DRx and #DB any more. */
9953 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
9954 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9955 AssertRCReturn(rc, rc);
9956
9957 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9958 {
9959#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9960 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
9961 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9962 AssertRCReturn(rc, rc);
9963#endif
9964 }
9965
9966 /* We're playing with the host CPU state here, make sure we can't preempt. */
9967 HM_DISABLE_PREEMPT_IF_NEEDED();
9968
9969 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9970 PVM pVM = pVCpu->CTX_SUFF(pVM);
9971 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9972 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
9973
9974 HM_RESTORE_PREEMPT_IF_NEEDED();
9975
9976#ifdef VBOX_WITH_STATISTICS
9977 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9978 AssertRCReturn(rc, rc);
9979 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9980 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9981 else
9982 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9983#endif
9984 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
9985 return VINF_SUCCESS;
9986 }
9987
9988 /*
9989 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
9990 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
9991 */
9992 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9993 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9994 AssertRCReturn(rc, rc);
9995 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9996
9997 PVM pVM = pVCpu->CTX_SUFF(pVM);
9998 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9999 {
10000 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10001 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
10002 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
10003 if (RT_SUCCESS(rc))
10004 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
10005 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
10006 }
10007 else
10008 {
10009 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10010 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
10011 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
10012 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
10013 }
10014
10015 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10016 if (RT_SUCCESS(rc))
10017 {
10018 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10019 AssertRCReturn(rc2, rc2);
10020 }
10021 return rc;
10022}
10023
10024
10025/**
10026 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10027 * Conditional VM-exit.
10028 */
10029HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10030{
10031 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10032 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10033
10034 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10035 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10036 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10037 return VINF_SUCCESS;
10038 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10039 return rc;
10040
10041 RTGCPHYS GCPhys = 0;
10042 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10043
10044#if 0
10045 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10046#else
10047 /* Aggressive state sync. for now. */
10048 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10049 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10050 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10051#endif
10052 AssertRCReturn(rc, rc);
10053
10054 /*
10055 * If we succeed, resume guest execution.
10056 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10057 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10058 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10059 * weird case. See @bugref{6043}.
10060 */
10061 PVM pVM = pVCpu->CTX_SUFF(pVM);
10062 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10063 rc = VBOXSTRICTRC_VAL(rc2);
10064 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10065 if ( rc == VINF_SUCCESS
10066 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10067 || rc == VERR_PAGE_NOT_PRESENT)
10068 {
10069 /* Successfully handled MMIO operation. */
10070 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10071 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10072 rc = VINF_SUCCESS;
10073 }
10074 return rc;
10075}
10076
10077
10078/**
10079 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10080 * VM-exit.
10081 */
10082HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10083{
10084 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10085 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10086
10087 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10088 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10089 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10090 return VINF_SUCCESS;
10091 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10092 return rc;
10093
10094 RTGCPHYS GCPhys = 0;
10095 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10096 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10097#if 0
10098 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10099#else
10100 /* Aggressive state sync. for now. */
10101 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10102 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10103 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10104#endif
10105 AssertRCReturn(rc, rc);
10106
10107 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10108 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10109
10110 RTGCUINT uErrorCode = 0;
10111 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10112 uErrorCode |= X86_TRAP_PF_ID;
10113 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10114 uErrorCode |= X86_TRAP_PF_RW;
10115 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10116 uErrorCode |= X86_TRAP_PF_P;
10117
10118 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10119
10120 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10121 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10122
10123 /* Handle the pagefault trap for the nested shadow table. */
10124 PVM pVM = pVCpu->CTX_SUFF(pVM);
10125 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10126 TRPMResetTrap(pVCpu);
10127
10128 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10129 if ( rc == VINF_SUCCESS
10130 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10131 || rc == VERR_PAGE_NOT_PRESENT)
10132 {
10133 /* Successfully synced our nested page tables. */
10134 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10135 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10136 return VINF_SUCCESS;
10137 }
10138
10139 Log4(("EPT return to ring-3 rc=%d\n"));
10140 return rc;
10141}
10142
10143/** @} */
10144
10145/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10146/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10147/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10148
10149/** @name VM-exit exception handlers.
10150 * @{
10151 */
10152
10153/**
10154 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10155 */
10156static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10157{
10158 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10159 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10160
10161 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10162 AssertRCReturn(rc, rc);
10163
10164 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10165 {
10166 /* Old-style FPU error reporting needs some extra work. */
10167 /** @todo don't fall back to the recompiler, but do it manually. */
10168 return VERR_EM_INTERPRETER;
10169 }
10170
10171 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10172 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10173 return rc;
10174}
10175
10176
10177/**
10178 * VM-exit exception handler for #BP (Breakpoint exception).
10179 */
10180static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10181{
10182 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10183 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10184
10185 /** @todo Try optimize this by not saving the entire guest state unless
10186 * really needed. */
10187 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10188 AssertRCReturn(rc, rc);
10189
10190 PVM pVM = pVCpu->CTX_SUFF(pVM);
10191 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10192 if (rc == VINF_EM_RAW_GUEST_TRAP)
10193 {
10194 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10195 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10196 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10197 AssertRCReturn(rc, rc);
10198
10199 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10200 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10201 }
10202
10203 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10204 return rc;
10205}
10206
10207
10208/**
10209 * VM-exit exception handler for #DB (Debug exception).
10210 */
10211static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10212{
10213 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10214 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10215 Log6(("XcptDB\n"));
10216
10217 /*
10218 * Get the DR6-like values from the exit qualification and pass it to DBGF
10219 * for processing.
10220 */
10221 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10222 AssertRCReturn(rc, rc);
10223
10224 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10225 uint64_t uDR6 = X86_DR6_INIT_VAL;
10226 uDR6 |= ( pVmxTransient->uExitQualification
10227 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10228
10229 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10230 if (rc == VINF_EM_RAW_GUEST_TRAP)
10231 {
10232 /*
10233 * The exception was for the guest. Update DR6, DR7.GD and
10234 * IA32_DEBUGCTL.LBR before forwarding it.
10235 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10236 */
10237 HM_DISABLE_PREEMPT_IF_NEEDED();
10238
10239 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10240 pMixedCtx->dr[6] |= uDR6;
10241 if (CPUMIsGuestDebugStateActive(pVCpu))
10242 ASMSetDR6(pMixedCtx->dr[6]);
10243
10244 HM_RESTORE_PREEMPT_IF_NEEDED();
10245
10246 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10247 AssertRCReturn(rc, rc);
10248
10249 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10250 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10251
10252 /* Paranoia. */
10253 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10254 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10255
10256 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10257 AssertRCReturn(rc, rc);
10258
10259 /*
10260 * Raise #DB in the guest.
10261 */
10262 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10263 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10264 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10265 AssertRCReturn(rc, rc);
10266 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10267 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10268 return VINF_SUCCESS;
10269 }
10270
10271 /*
10272 * Not a guest trap, must be a hypervisor related debug event then.
10273 * Update DR6 in case someone is interested in it.
10274 */
10275 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10276 AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
10277 CPUMSetHyperDR6(pVCpu, uDR6);
10278
10279 return rc;
10280}
10281
10282
10283/**
10284 * VM-exit exception handler for #NM (Device-not-available exception: floating
10285 * point exception).
10286 */
10287static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10288{
10289 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10290
10291#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10292 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
10293#endif
10294
10295 /* We require CR0 and EFER. EFER is always up-to-date. */
10296 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10297 AssertRCReturn(rc, rc);
10298
10299 /* We're playing with the host CPU state here, have to disable preemption. */
10300 HM_DISABLE_PREEMPT_IF_NEEDED();
10301
10302 /* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10303 PVM pVM = pVCpu->CTX_SUFF(pVM);
10304 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
10305 if (rc == VINF_SUCCESS)
10306 {
10307 Assert(CPUMIsGuestFPUStateActive(pVCpu));
10308 HM_RESTORE_PREEMPT_IF_NEEDED();
10309
10310 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
10311 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10312 return VINF_SUCCESS;
10313 }
10314
10315 HM_RESTORE_PREEMPT_IF_NEEDED();
10316
10317 /* Forward #NM to the guest. */
10318 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10319 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10320 AssertRCReturn(rc, rc);
10321 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10322 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10323 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10324 return rc;
10325}
10326
10327
10328/**
10329 * VM-exit exception handler for #GP (General-protection exception).
10330 *
10331 * @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
10332 */
10333static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10334{
10335 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10336 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10337
10338 int rc = VERR_INTERNAL_ERROR_5;
10339 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10340 {
10341#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10342 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10343 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10344 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10345 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10346 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10347 AssertRCReturn(rc, rc);
10348 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10349 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10350 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10351 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10352 return rc;
10353#else
10354 /* We don't intercept #GP. */
10355 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10356 return VERR_VMX_UNEXPECTED_EXCEPTION;
10357#endif
10358 }
10359
10360 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10361 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10362
10363 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10364 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10365 AssertRCReturn(rc, rc);
10366
10367 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10368 uint32_t cbOp = 0;
10369 PVM pVM = pVCpu->CTX_SUFF(pVM);
10370 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10371 if (RT_SUCCESS(rc))
10372 {
10373 rc = VINF_SUCCESS;
10374 Assert(cbOp == pDis->cbInstr);
10375 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10376 switch (pDis->pCurInstr->uOpcode)
10377 {
10378 case OP_CLI:
10379 {
10380 pMixedCtx->eflags.Bits.u1IF = 0;
10381 pMixedCtx->rip += pDis->cbInstr;
10382 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10383 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10384 break;
10385 }
10386
10387 case OP_STI:
10388 {
10389 pMixedCtx->eflags.Bits.u1IF = 1;
10390 pMixedCtx->rip += pDis->cbInstr;
10391 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10392 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10393 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10394 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10395 break;
10396 }
10397
10398 case OP_HLT:
10399 {
10400 rc = VINF_EM_HALT;
10401 pMixedCtx->rip += pDis->cbInstr;
10402 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
10403 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10404 break;
10405 }
10406
10407 case OP_POPF:
10408 {
10409 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10410 uint32_t cbParm = 0;
10411 uint32_t uMask = 0;
10412 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10413 {
10414 cbParm = 4;
10415 uMask = 0xffffffff;
10416 }
10417 else
10418 {
10419 cbParm = 2;
10420 uMask = 0xffff;
10421 }
10422
10423 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10424 RTGCPTR GCPtrStack = 0;
10425 X86EFLAGS Eflags;
10426 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10427 &GCPtrStack);
10428 if (RT_SUCCESS(rc))
10429 {
10430 Assert(sizeof(Eflags.u32) >= cbParm);
10431 Eflags.u32 = 0;
10432 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10433 }
10434 if (RT_FAILURE(rc))
10435 {
10436 rc = VERR_EM_INTERPRETER;
10437 break;
10438 }
10439 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10440 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10441 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10442 /* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10443 pMixedCtx->eflags.Bits.u1RF = 0;
10444 pMixedCtx->esp += cbParm;
10445 pMixedCtx->esp &= uMask;
10446 pMixedCtx->rip += pDis->cbInstr;
10447 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10448 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10449 break;
10450 }
10451
10452 case OP_PUSHF:
10453 {
10454 uint32_t cbParm = 0;
10455 uint32_t uMask = 0;
10456 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10457 {
10458 cbParm = 4;
10459 uMask = 0xffffffff;
10460 }
10461 else
10462 {
10463 cbParm = 2;
10464 uMask = 0xffff;
10465 }
10466
10467 /* Get the stack pointer & push the contents of eflags onto the stack. */
10468 RTGCPTR GCPtrStack = 0;
10469 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10470 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10471 if (RT_FAILURE(rc))
10472 {
10473 rc = VERR_EM_INTERPRETER;
10474 break;
10475 }
10476 X86EFLAGS Eflags = pMixedCtx->eflags;
10477 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10478 Eflags.Bits.u1RF = 0;
10479 Eflags.Bits.u1VM = 0;
10480
10481 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10482 if (RT_FAILURE(rc))
10483 {
10484 rc = VERR_EM_INTERPRETER;
10485 break;
10486 }
10487 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10488 pMixedCtx->esp -= cbParm;
10489 pMixedCtx->esp &= uMask;
10490 pMixedCtx->rip += pDis->cbInstr;
10491 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP;
10492 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10493 break;
10494 }
10495
10496 case OP_IRET:
10497 {
10498 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10499 * instruction reference. */
10500 RTGCPTR GCPtrStack = 0;
10501 uint32_t uMask = 0xffff;
10502 uint16_t aIretFrame[3];
10503 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10504 {
10505 rc = VERR_EM_INTERPRETER;
10506 break;
10507 }
10508 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10509 &GCPtrStack);
10510 if (RT_SUCCESS(rc))
10511 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10512 if (RT_FAILURE(rc))
10513 {
10514 rc = VERR_EM_INTERPRETER;
10515 break;
10516 }
10517 pMixedCtx->eip = 0;
10518 pMixedCtx->ip = aIretFrame[0];
10519 pMixedCtx->cs.Sel = aIretFrame[1];
10520 pMixedCtx->cs.ValidSel = aIretFrame[1];
10521 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10522 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10523 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10524 pMixedCtx->sp += sizeof(aIretFrame);
10525 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_SEGMENT_REGS | HM_CHANGED_GUEST_RSP
10526 | HM_CHANGED_GUEST_RFLAGS;
10527 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10528 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10529 break;
10530 }
10531
10532 case OP_INT:
10533 {
10534 uint16_t uVector = pDis->Param1.uValue & 0xff;
10535 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10536 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10537 break;
10538 }
10539
10540 case OP_INTO:
10541 {
10542 if (pMixedCtx->eflags.Bits.u1OF)
10543 {
10544 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10545 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10546 }
10547 break;
10548 }
10549
10550 default:
10551 {
10552 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10553 EMCODETYPE_SUPERVISOR);
10554 rc = VBOXSTRICTRC_VAL(rc2);
10555 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
10556 Log4(("#GP rc=%Rrc\n", rc));
10557 break;
10558 }
10559 }
10560 }
10561 else
10562 rc = VERR_EM_INTERPRETER;
10563
10564 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10565 ("#GP Unexpected rc=%Rrc\n", rc));
10566 return rc;
10567}
10568
10569
10570/**
10571 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10572 * the exception reported in the VMX transient structure back into the VM.
10573 *
10574 * @remarks Requires uExitIntrInfo in the VMX transient structure to be
10575 * up-to-date.
10576 */
10577static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10578{
10579 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10580
10581 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10582 hmR0VmxCheckExitDueToEventDelivery(). */
10583 int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10584 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10585 AssertRCReturn(rc, rc);
10586 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10587
10588 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10589 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10590 return VINF_SUCCESS;
10591}
10592
10593
10594/**
10595 * VM-exit exception handler for #PF (Page-fault exception).
10596 */
10597static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10598{
10599 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10600 PVM pVM = pVCpu->CTX_SUFF(pVM);
10601 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10602 rc |= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10603 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10604 AssertRCReturn(rc, rc);
10605
10606#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
10607 if (pVM->hm.s.fNestedPaging)
10608 {
10609 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10610 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10611 {
10612 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10613 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10614 0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10615 }
10616 else
10617 {
10618 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10619 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10620 Log4(("Pending #DF due to vectoring #PF. NP\n"));
10621 }
10622 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10623 return rc;
10624 }
10625#else
10626 Assert(!pVM->hm.s.fNestedPaging);
10627#endif
10628
10629 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10630 AssertRCReturn(rc, rc);
10631
10632 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10633 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10634
10635 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10636 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10637 (RTGCPTR)pVmxTransient->uExitQualification);
10638
10639 Log4(("#PF: rc=%Rrc\n", rc));
10640 if (rc == VINF_SUCCESS)
10641 {
10642 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10643 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
10644 * memory? We don't update the whole state here... */
10645 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10646 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10647 TRPMResetTrap(pVCpu);
10648 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10649 return rc;
10650 }
10651 else if (rc == VINF_EM_RAW_GUEST_TRAP)
10652 {
10653 if (!pVmxTransient->fVectoringPF)
10654 {
10655 /* It's a guest page fault and needs to be reflected to the guest. */
10656 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10657 TRPMResetTrap(pVCpu);
10658 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10659 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10660 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10661 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10662 }
10663 else
10664 {
10665 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10666 TRPMResetTrap(pVCpu);
10667 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10668 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10669 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10670 }
10671
10672 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10673 return VINF_SUCCESS;
10674 }
10675
10676 TRPMResetTrap(pVCpu);
10677 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10678 return rc;
10679}
10680
10681/** @} */
10682
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette