VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 48006

Last change on this file since 48006 was 47999, checked in by vboxsync, 11 years ago

VMM/HM: Better error reporting for unsupported VT-x feature combos.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 440.9 KB
Line 
1/* $Id: HMVMXR0.cpp 47999 2013-08-22 16:03:03Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#endif
45
46
47/*******************************************************************************
48* Defined Constants And Macros *
49*******************************************************************************/
50#if defined(RT_ARCH_AMD64)
51# define HMVMX_IS_64BIT_HOST_MODE() (true)
52typedef RTHCUINTREG HMVMXHCUINTREG;
53#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54extern "C" uint32_t g_fVMXIs64bitHost;
55# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56typedef uint64_t HMVMXHCUINTREG;
57#else
58# define HMVMX_IS_64BIT_HOST_MODE() (false)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#endif
61
62/** Use the function table. */
63#define HMVMX_USE_FUNCTION_TABLE
64
65/** Determine which tagged-TLB flush handler to use. */
66#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71/** @name Updated-guest-state flags.
72 * @{ */
73#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94 | HMVMX_UPDATED_GUEST_RSP \
95 | HMVMX_UPDATED_GUEST_RFLAGS \
96 | HMVMX_UPDATED_GUEST_CR0 \
97 | HMVMX_UPDATED_GUEST_CR3 \
98 | HMVMX_UPDATED_GUEST_CR4 \
99 | HMVMX_UPDATED_GUEST_GDTR \
100 | HMVMX_UPDATED_GUEST_IDTR \
101 | HMVMX_UPDATED_GUEST_LDTR \
102 | HMVMX_UPDATED_GUEST_TR \
103 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104 | HMVMX_UPDATED_GUEST_DEBUG \
105 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112 | HMVMX_UPDATED_GUEST_APIC_STATE)
113/** @} */
114
115/** @name
116 * Flags to skip redundant reads of some common VMCS fields that are not part of
117 * the guest-CPU state but are in the transient structure.
118 */
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually (except #PF). #NM is also
143 * handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
144 * intercepted even in real-mode if we have Nested Paging support.
145 */
146#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
147 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
148 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
149 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
150 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
151 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
152 | RT_BIT(X86_XCPT_XF))
153
154/**
155 * Exception bitmap mask for all contributory exceptions.
156 *
157 * Page fault is deliberately excluded here as it's conditional as to whether
158 * it's contributory or benign. Page faults are handled separately.
159 */
160#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
161 | RT_BIT(X86_XCPT_DE))
162
163/** Maximum VM-instruction error number. */
164#define HMVMX_INSTR_ERROR_MAX 28
165
166/** Profiling macro. */
167#ifdef HM_PROFILE_EXIT_DISPATCH
168# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
169# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
170#else
171# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
173#endif
174
175/** Assert that preemption is disabled or covered by thread-context hooks. */
176#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
177 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
178
179/** Assert that we haven't migrated CPUs when thread-context hooks are not
180 * used. */
181#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
182 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
183 ("Illegal migration! Entered on CPU %u Current %u\n", \
184 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
185
186/*******************************************************************************
187* Structures and Typedefs *
188*******************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG uEflags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
203 uint64_t u64LStarMsr;
204 /** The guest's TPR value used for TPR shadowing. */
205 uint8_t u8GuestTpr;
206 /** Alignment. */
207 uint8_t abAlignment0[7];
208
209 /** The basic VM-exit reason. */
210 uint16_t uExitReason;
211 /** Alignment. */
212 uint16_t u16Alignment0;
213 /** The VM-exit interruption error code. */
214 uint32_t uExitIntrErrorCode;
215 /** The VM-exit exit qualification. */
216 uint64_t uExitQualification;
217
218 /** The VM-exit interruption-information field. */
219 uint32_t uExitIntrInfo;
220 /** The VM-exit instruction-length field. */
221 uint32_t cbInstr;
222 /** The VM-exit instruction-information field. */
223 union
224 {
225 /** Plain unsigned int representation. */
226 uint32_t u;
227 /** INS and OUTS information. */
228 struct
229 {
230 uint32_t u6Reserved0 : 6;
231 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
232 uint32_t u3AddrSize : 3;
233 uint32_t u5Reserved1 : 5;
234 /** The segment register (X86_SREG_XXX). */
235 uint32_t iSegReg : 3;
236 uint32_t uReserved2 : 14;
237 } StrIo;
238 } ExitInstrInfo;
239 /** Whether the VM-entry failed or not. */
240 bool fVMEntryFailed;
241 /** Alignment. */
242 uint8_t abAlignment1[3];
243
244 /** The VM-entry interruption-information field. */
245 uint32_t uEntryIntrInfo;
246 /** The VM-entry exception error code field. */
247 uint32_t uEntryXcptErrorCode;
248 /** The VM-entry instruction length field. */
249 uint32_t cbEntryInstr;
250
251 /** IDT-vectoring information field. */
252 uint32_t uIdtVectoringInfo;
253 /** IDT-vectoring error code. */
254 uint32_t uIdtVectoringErrorCode;
255
256 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
257 uint32_t fVmcsFieldsRead;
258 /** Whether TSC-offsetting should be setup before VM-entry. */
259 bool fUpdateTscOffsettingAndPreemptTimer;
260 /** Whether the VM-exit was caused by a page-fault during delivery of a
261 * contributory exception or a page-fault. */
262 bool fVectoringPF;
263} VMXTRANSIENT;
264AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
265AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
266AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
267AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
268/** Pointer to VMX transient state. */
269typedef VMXTRANSIENT *PVMXTRANSIENT;
270
271
272/**
273 * MSR-bitmap read permissions.
274 */
275typedef enum VMXMSREXITREAD
276{
277 /** Reading this MSR causes a VM-exit. */
278 VMXMSREXIT_INTERCEPT_READ = 0xb,
279 /** Reading this MSR does not cause a VM-exit. */
280 VMXMSREXIT_PASSTHRU_READ
281} VMXMSREXITREAD;
282
283/**
284 * MSR-bitmap write permissions.
285 */
286typedef enum VMXMSREXITWRITE
287{
288 /** Writing to this MSR causes a VM-exit. */
289 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
290 /** Writing to this MSR does not cause a VM-exit. */
291 VMXMSREXIT_PASSTHRU_WRITE
292} VMXMSREXITWRITE;
293
294/**
295 * VM-exit handler.
296 *
297 * @returns VBox status code.
298 * @param pVCpu Pointer to the VMCPU.
299 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
300 * out-of-sync. Make sure to update the required
301 * fields before using them.
302 * @param pVmxTransient Pointer to the VMX-transient structure.
303 */
304#ifndef HMVMX_USE_FUNCTION_TABLE
305typedef int FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306#else
307typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308/** Pointer to VM-exit handler. */
309typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
310#endif
311
312
313/*******************************************************************************
314* Internal Functions *
315*******************************************************************************/
316static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
317static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
318static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
319 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
320#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
321static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
322#endif
323#ifndef HMVMX_USE_FUNCTION_TABLE
324DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
325# define HMVMX_EXIT_DECL static int
326#else
327# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
328#endif
329
330/** @name VM-exit handlers.
331 * @{
332 */
333static FNVMEXITHANDLER hmR0VmxExitXcptOrNmi;
334static FNVMEXITHANDLER hmR0VmxExitExtInt;
335static FNVMEXITHANDLER hmR0VmxExitTripleFault;
336static FNVMEXITHANDLER hmR0VmxExitInitSignal;
337static FNVMEXITHANDLER hmR0VmxExitSipi;
338static FNVMEXITHANDLER hmR0VmxExitIoSmi;
339static FNVMEXITHANDLER hmR0VmxExitSmi;
340static FNVMEXITHANDLER hmR0VmxExitIntWindow;
341static FNVMEXITHANDLER hmR0VmxExitNmiWindow;
342static FNVMEXITHANDLER hmR0VmxExitTaskSwitch;
343static FNVMEXITHANDLER hmR0VmxExitCpuid;
344static FNVMEXITHANDLER hmR0VmxExitGetsec;
345static FNVMEXITHANDLER hmR0VmxExitHlt;
346static FNVMEXITHANDLER hmR0VmxExitInvd;
347static FNVMEXITHANDLER hmR0VmxExitInvlpg;
348static FNVMEXITHANDLER hmR0VmxExitRdpmc;
349static FNVMEXITHANDLER hmR0VmxExitRdtsc;
350static FNVMEXITHANDLER hmR0VmxExitRsm;
351static FNVMEXITHANDLER hmR0VmxExitSetPendingXcptUD;
352static FNVMEXITHANDLER hmR0VmxExitMovCRx;
353static FNVMEXITHANDLER hmR0VmxExitMovDRx;
354static FNVMEXITHANDLER hmR0VmxExitIoInstr;
355static FNVMEXITHANDLER hmR0VmxExitRdmsr;
356static FNVMEXITHANDLER hmR0VmxExitWrmsr;
357static FNVMEXITHANDLER hmR0VmxExitErrInvalidGuestState;
358static FNVMEXITHANDLER hmR0VmxExitErrMsrLoad;
359static FNVMEXITHANDLER hmR0VmxExitErrUndefined;
360static FNVMEXITHANDLER hmR0VmxExitMwait;
361static FNVMEXITHANDLER hmR0VmxExitMtf;
362static FNVMEXITHANDLER hmR0VmxExitMonitor;
363static FNVMEXITHANDLER hmR0VmxExitPause;
364static FNVMEXITHANDLER hmR0VmxExitErrMachineCheck;
365static FNVMEXITHANDLER hmR0VmxExitTprBelowThreshold;
366static FNVMEXITHANDLER hmR0VmxExitApicAccess;
367static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
368static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
369static FNVMEXITHANDLER hmR0VmxExitEptViolation;
370static FNVMEXITHANDLER hmR0VmxExitEptMisconfig;
371static FNVMEXITHANDLER hmR0VmxExitRdtscp;
372static FNVMEXITHANDLER hmR0VmxExitPreemptTimer;
373static FNVMEXITHANDLER hmR0VmxExitWbinvd;
374static FNVMEXITHANDLER hmR0VmxExitXsetbv;
375static FNVMEXITHANDLER hmR0VmxExitRdrand;
376static FNVMEXITHANDLER hmR0VmxExitInvpcid;
377/** @} */
378
379static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
380static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
381static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
382static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
383static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
384static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
385static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
386static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
387
388/*******************************************************************************
389* Global Variables *
390*******************************************************************************/
391#ifdef HMVMX_USE_FUNCTION_TABLE
392
393/**
394 * VMX_EXIT dispatch table.
395 */
396static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
397{
398 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
399 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
400 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
401 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
402 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
403 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
404 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
405 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
406 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
407 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
408 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
409 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
410 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
411 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
412 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
413 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
414 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
415 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
416 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
417 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
418 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
419 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
420 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
421 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
422 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
423 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
424 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
425 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
426 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
427 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
428 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
429 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
430 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
431 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
432 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
433 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
434 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
435 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
436 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
437 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
438 /* 40 UNDEFINED */ hmR0VmxExitPause,
439 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
440 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
441 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
442 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
443 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
444 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
445 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
446 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
447 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
448 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
449 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
450 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
451 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
452 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
453 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
454 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
455 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
456 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
457 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
458};
459#endif /* HMVMX_USE_FUNCTION_TABLE */
460
461#ifdef VBOX_STRICT
462static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
463{
464 /* 0 */ "(Not Used)",
465 /* 1 */ "VMCALL executed in VMX root operation.",
466 /* 2 */ "VMCLEAR with invalid physical address.",
467 /* 3 */ "VMCLEAR with VMXON pointer.",
468 /* 4 */ "VMLAUNCH with non-clear VMCS.",
469 /* 5 */ "VMRESUME with non-launched VMCS.",
470 /* 6 */ "VMRESUME after VMXOFF",
471 /* 7 */ "VM entry with invalid control fields.",
472 /* 8 */ "VM entry with invalid host state fields.",
473 /* 9 */ "VMPTRLD with invalid physical address.",
474 /* 10 */ "VMPTRLD with VMXON pointer.",
475 /* 11 */ "VMPTRLD with incorrect revision identifier.",
476 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
477 /* 13 */ "VMWRITE to read-only VMCS component.",
478 /* 14 */ "(Not Used)",
479 /* 15 */ "VMXON executed in VMX root operation.",
480 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
481 /* 17 */ "VM entry with non-launched executing VMCS.",
482 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
483 /* 19 */ "VMCALL with non-clear VMCS.",
484 /* 20 */ "VMCALL with invalid VM-exit control fields.",
485 /* 21 */ "(Not Used)",
486 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
487 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
488 /* 24 */ "VMCALL with invalid SMM-monitor features.",
489 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
490 /* 26 */ "VM entry with events blocked by MOV SS.",
491 /* 27 */ "(Not Used)",
492 /* 28 */ "Invalid operand to INVEPT/INVVPID."
493};
494#endif /* VBOX_STRICT */
495
496
497
498/**
499 * Updates the VM's last error record. If there was a VMX instruction error,
500 * reads the error data from the VMCS and updates VCPU's last error record as
501 * well.
502 *
503 * @param pVM Pointer to the VM.
504 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
505 * VERR_VMX_UNABLE_TO_START_VM or
506 * VERR_VMX_INVALID_VMCS_FIELD).
507 * @param rc The error code.
508 */
509static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
510{
511 AssertPtr(pVM);
512 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
513 || rc == VERR_VMX_UNABLE_TO_START_VM)
514 {
515 AssertPtrReturnVoid(pVCpu);
516 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
517 }
518 pVM->hm.s.lLastError = rc;
519}
520
521
522/**
523 * Reads the VM-entry interruption-information field from the VMCS into the VMX
524 * transient structure.
525 *
526 * @returns VBox status code.
527 * @param pVmxTransient Pointer to the VMX transient structure.
528 *
529 * @remarks No-long-jump zone!!!
530 */
531DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
532{
533 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
534 AssertRCReturn(rc, rc);
535 return VINF_SUCCESS;
536}
537
538
539/**
540 * Reads the VM-entry exception error code field from the VMCS into
541 * the VMX transient structure.
542 *
543 * @returns VBox status code.
544 * @param pVmxTransient Pointer to the VMX transient structure.
545 *
546 * @remarks No-long-jump zone!!!
547 */
548DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
549{
550 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
551 AssertRCReturn(rc, rc);
552 return VINF_SUCCESS;
553}
554
555
556/**
557 * Reads the VM-entry exception error code field from the VMCS into
558 * the VMX transient structure.
559 *
560 * @returns VBox status code.
561 * @param pVCpu Pointer to the VMCPU.
562 * @param pVmxTransient Pointer to the VMX transient structure.
563 *
564 * @remarks No-long-jump zone!!!
565 */
566DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
567{
568 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
569 AssertRCReturn(rc, rc);
570 return VINF_SUCCESS;
571}
572
573
574/**
575 * Reads the VM-exit interruption-information field from the VMCS into the VMX
576 * transient structure.
577 *
578 * @returns VBox status code.
579 * @param pVCpu Pointer to the VMCPU.
580 * @param pVmxTransient Pointer to the VMX transient structure.
581 */
582DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
583{
584 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
585 {
586 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
587 AssertRCReturn(rc, rc);
588 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
589 }
590 return VINF_SUCCESS;
591}
592
593
594/**
595 * Reads the VM-exit interruption error code from the VMCS into the VMX
596 * transient structure.
597 *
598 * @returns VBox status code.
599 * @param pVCpu Pointer to the VMCPU.
600 * @param pVmxTransient Pointer to the VMX transient structure.
601 */
602DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
603{
604 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
605 {
606 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
607 AssertRCReturn(rc, rc);
608 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
609 }
610 return VINF_SUCCESS;
611}
612
613
614/**
615 * Reads the VM-exit instruction length field from the VMCS into the VMX
616 * transient structure.
617 *
618 * @returns VBox status code.
619 * @param pVCpu Pointer to the VMCPU.
620 * @param pVmxTransient Pointer to the VMX transient structure.
621 */
622DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
623{
624 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
625 {
626 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
627 AssertRCReturn(rc, rc);
628 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
629 }
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit instruction-information field from the VMCS into
636 * the VMX transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVCpu The cross context per CPU structure.
640 * @param pVmxTransient Pointer to the VMX transient structure.
641 */
642DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
643{
644 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645 {
646 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
647 AssertRCReturn(rc, rc);
648 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649 }
650 return VINF_SUCCESS;
651}
652
653
654/**
655 * Reads the exit qualification from the VMCS into the VMX transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVCpu Pointer to the VMCPU.
659 * @param pVmxTransient Pointer to the VMX transient structure.
660 */
661DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
662{
663 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
664 {
665 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
666 AssertRCReturn(rc, rc);
667 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
668 }
669 return VINF_SUCCESS;
670}
671
672
673/**
674 * Reads the IDT-vectoring information field from the VMCS into the VMX
675 * transient structure.
676 *
677 * @returns VBox status code.
678 * @param pVmxTransient Pointer to the VMX transient structure.
679 *
680 * @remarks No-long-jump zone!!!
681 */
682DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
683{
684 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
685 {
686 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
687 AssertRCReturn(rc, rc);
688 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
689 }
690 return VINF_SUCCESS;
691}
692
693
694/**
695 * Reads the IDT-vectoring error code from the VMCS into the VMX
696 * transient structure.
697 *
698 * @returns VBox status code.
699 * @param pVmxTransient Pointer to the VMX transient structure.
700 */
701DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
702{
703 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
704 {
705 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
706 AssertRCReturn(rc, rc);
707 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
708 }
709 return VINF_SUCCESS;
710}
711
712
713/**
714 * Enters VMX root mode operation on the current CPU.
715 *
716 * @returns VBox status code.
717 * @param pVM Pointer to the VM (optional, can be NULL, after
718 * a resume).
719 * @param HCPhysCpuPage Physical address of the VMXON region.
720 * @param pvCpuPage Pointer to the VMXON region.
721 */
722static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
723{
724 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
725 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
726 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
727
728 if (pVM)
729 {
730 /* Write the VMCS revision dword to the VMXON region. */
731 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
732 }
733
734 /* Enable the VMX bit in CR4 if necessary. */
735 RTCCUINTREG uCr4 = ASMGetCR4();
736 if (!(uCr4 & X86_CR4_VMXE))
737 ASMSetCR4(uCr4 | X86_CR4_VMXE);
738
739 /* Enter VMX root mode. */
740 int rc = VMXEnable(HCPhysCpuPage);
741 if (RT_FAILURE(rc))
742 ASMSetCR4(uCr4);
743
744 return rc;
745}
746
747
748/**
749 * Exits VMX root mode operation on the current CPU.
750 *
751 * @returns VBox status code.
752 */
753static int hmR0VmxLeaveRootMode(void)
754{
755 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
756
757 /* If we're for some reason not in VMX root mode, then don't leave it. */
758 RTCCUINTREG uHostCR4 = ASMGetCR4();
759 if (uHostCR4 & X86_CR4_VMXE)
760 {
761 /* Exit VMX root mode and clear the VMX bit in CR4. */
762 VMXDisable();
763 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
764 return VINF_SUCCESS;
765 }
766
767 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
768}
769
770
771/**
772 * Allocates and maps one physically contiguous page. The allocated page is
773 * zero'd out. (Used by various VT-x structures).
774 *
775 * @returns IPRT status code.
776 * @param pMemObj Pointer to the ring-0 memory object.
777 * @param ppVirt Where to store the virtual address of the
778 * allocation.
779 * @param pPhys Where to store the physical address of the
780 * allocation.
781 */
782DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
783{
784 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
785 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
786 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
787
788 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
789 if (RT_FAILURE(rc))
790 return rc;
791 *ppVirt = RTR0MemObjAddress(*pMemObj);
792 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
793 ASMMemZero32(*ppVirt, PAGE_SIZE);
794 return VINF_SUCCESS;
795}
796
797
798/**
799 * Frees and unmaps an allocated physical page.
800 *
801 * @param pMemObj Pointer to the ring-0 memory object.
802 * @param ppVirt Where to re-initialize the virtual address of
803 * allocation as 0.
804 * @param pHCPhys Where to re-initialize the physical address of the
805 * allocation as 0.
806 */
807DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
808{
809 AssertPtr(pMemObj);
810 AssertPtr(ppVirt);
811 AssertPtr(pHCPhys);
812 if (*pMemObj != NIL_RTR0MEMOBJ)
813 {
814 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
815 AssertRC(rc);
816 *pMemObj = NIL_RTR0MEMOBJ;
817 *ppVirt = 0;
818 *pHCPhys = 0;
819 }
820}
821
822
823/**
824 * Worker function to free VT-x related structures.
825 *
826 * @returns IPRT status code.
827 * @param pVM Pointer to the VM.
828 */
829static void hmR0VmxStructsFree(PVM pVM)
830{
831 for (VMCPUID i = 0; i < pVM->cCpus; i++)
832 {
833 PVMCPU pVCpu = &pVM->aCpus[i];
834 AssertPtr(pVCpu);
835
836#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
837 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
838 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
839#endif
840
841 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
842 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
843
844 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
845 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
846 }
847
848 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
849#ifdef VBOX_WITH_CRASHDUMP_MAGIC
850 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
851#endif
852}
853
854
855/**
856 * Worker function to allocate VT-x related VM structures.
857 *
858 * @returns IPRT status code.
859 * @param pVM Pointer to the VM.
860 */
861static int hmR0VmxStructsAlloc(PVM pVM)
862{
863 /*
864 * Initialize members up-front so we can cleanup properly on allocation failure.
865 */
866#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
867 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
868 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
869 pVM->hm.s.vmx.HCPhys##a_Name = 0;
870
871#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
872 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
873 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
874 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
875
876#ifdef VBOX_WITH_CRASHDUMP_MAGIC
877 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
878#endif
879 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
880
881 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
882 for (VMCPUID i = 0; i < pVM->cCpus; i++)
883 {
884 PVMCPU pVCpu = &pVM->aCpus[i];
885 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
886 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
887 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
888#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
889 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
890 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
891#endif
892 }
893#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
894#undef VMXLOCAL_INIT_VM_MEMOBJ
895
896 /*
897 * Allocate all the VT-x structures.
898 */
899 int rc = VINF_SUCCESS;
900#ifdef VBOX_WITH_CRASHDUMP_MAGIC
901 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
902 if (RT_FAILURE(rc))
903 goto cleanup;
904 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
905 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
906#endif
907
908 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
909 if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
910 {
911 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
912 &pVM->hm.s.vmx.HCPhysApicAccess);
913 if (RT_FAILURE(rc))
914 goto cleanup;
915 }
916
917 /*
918 * Initialize per-VCPU VT-x structures.
919 */
920 for (VMCPUID i = 0; i < pVM->cCpus; i++)
921 {
922 PVMCPU pVCpu = &pVM->aCpus[i];
923 AssertPtr(pVCpu);
924
925 /* Allocate the VM control structure (VMCS). */
926 AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
927 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
928 if (RT_FAILURE(rc))
929 goto cleanup;
930
931 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
932 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
933 {
934 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
935 &pVCpu->hm.s.vmx.HCPhysVirtApic);
936 if (RT_FAILURE(rc))
937 goto cleanup;
938 }
939
940 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
941 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
942 {
943 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
944 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
945 if (RT_FAILURE(rc))
946 goto cleanup;
947 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
948 }
949
950#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
951 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
952 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
953 if (RT_FAILURE(rc))
954 goto cleanup;
955
956 /* Allocate the VM-exit MSR-load page for the host MSRs. */
957 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
958 if (RT_FAILURE(rc))
959 goto cleanup;
960#endif
961 }
962
963 return VINF_SUCCESS;
964
965cleanup:
966 hmR0VmxStructsFree(pVM);
967 return rc;
968}
969
970
971/**
972 * Does global VT-x initialization (called during module initialization).
973 *
974 * @returns VBox status code.
975 */
976VMMR0DECL(int) VMXR0GlobalInit(void)
977{
978#ifdef HMVMX_USE_FUNCTION_TABLE
979 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
980# ifdef VBOX_STRICT
981 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
982 Assert(g_apfnVMExitHandlers[i]);
983# endif
984#endif
985 return VINF_SUCCESS;
986}
987
988
989/**
990 * Does global VT-x termination (called during module termination).
991 */
992VMMR0DECL(void) VMXR0GlobalTerm()
993{
994 /* Nothing to do currently. */
995}
996
997
998/**
999 * Sets up and activates VT-x on the current CPU.
1000 *
1001 * @returns VBox status code.
1002 * @param pCpu Pointer to the global CPU info struct.
1003 * @param pVM Pointer to the VM (can be NULL after a host resume
1004 * operation).
1005 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1006 * fEnabledByHost is true).
1007 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1008 * @a fEnabledByHost is true).
1009 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1010 * enable VT-x on the host.
1011 */
1012VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
1013{
1014 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1015 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1016
1017 if (!fEnabledByHost)
1018 {
1019 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1020 if (RT_FAILURE(rc))
1021 return rc;
1022 }
1023
1024 /*
1025 * Flush all EPTP tagged-TLB entries (in case any other hypervisor have been using EPTPs) so that
1026 * we can avoid an explicit flush while using new VPIDs. We would still need to flush
1027 * each time while reusing a VPID after hitting the MaxASID limit once.
1028 */
1029 if ( pVM
1030 && pVM->hm.s.fNestedPaging)
1031 {
1032 /* We require ALL_CONTEXT flush-type to be available on the CPU. See hmR0VmxSetupTaggedTlb(). */
1033 Assert(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS);
1034 hmR0VmxFlushEpt(pVM, NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1035 pCpu->fFlushAsidBeforeUse = false;
1036 }
1037 else
1038 {
1039 /** @todo This is still not perfect. If on host resume (pVM is NULL or a VM
1040 * without Nested Paging triggered this function) we still have the risk
1041 * of potentially running with stale TLB-entries from other hypervisors
1042 * when later we use a VM with NestedPaging. To fix this properly we will
1043 * have to pass '&g_HvmR0' (see HMR0.cpp) to this function and read
1044 * 'vmx_ept_vpid_caps' from it. Sigh. */
1045 pCpu->fFlushAsidBeforeUse = true;
1046 }
1047
1048 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1049 ++pCpu->cTlbFlushes;
1050
1051 return VINF_SUCCESS;
1052}
1053
1054
1055/**
1056 * Deactivates VT-x on the current CPU.
1057 *
1058 * @returns VBox status code.
1059 * @param pCpu Pointer to the global CPU info struct.
1060 * @param pvCpuPage Pointer to the VMXON region.
1061 * @param HCPhysCpuPage Physical address of the VMXON region.
1062 *
1063 * @remarks This function should never be called when SUPR0EnableVTx() or
1064 * similar was used to enable VT-x on the host.
1065 */
1066VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1067{
1068 NOREF(pCpu);
1069 NOREF(pvCpuPage);
1070 NOREF(HCPhysCpuPage);
1071
1072 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1073 return hmR0VmxLeaveRootMode();
1074}
1075
1076
1077/**
1078 * Sets the permission bits for the specified MSR in the MSR bitmap.
1079 *
1080 * @param pVCpu Pointer to the VMCPU.
1081 * @param uMSR The MSR value.
1082 * @param enmRead Whether reading this MSR causes a VM-exit.
1083 * @param enmWrite Whether writing this MSR causes a VM-exit.
1084 */
1085static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1086{
1087 int32_t iBit;
1088 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1089
1090 /*
1091 * Layout:
1092 * 0x000 - 0x3ff - Low MSR read bits
1093 * 0x400 - 0x7ff - High MSR read bits
1094 * 0x800 - 0xbff - Low MSR write bits
1095 * 0xc00 - 0xfff - High MSR write bits
1096 */
1097 if (uMsr <= 0x00001FFF)
1098 iBit = uMsr;
1099 else if ( uMsr >= 0xC0000000
1100 && uMsr <= 0xC0001FFF)
1101 {
1102 iBit = (uMsr - 0xC0000000);
1103 pbMsrBitmap += 0x400;
1104 }
1105 else
1106 {
1107 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1108 return;
1109 }
1110
1111 Assert(iBit <= 0x1fff);
1112 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1113 ASMBitSet(pbMsrBitmap, iBit);
1114 else
1115 ASMBitClear(pbMsrBitmap, iBit);
1116
1117 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1118 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1119 else
1120 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1121}
1122
1123
1124/**
1125 * Flushes the TLB using EPT.
1126 *
1127 * @returns VBox status code.
1128 * @param pVM Pointer to the VM.
1129 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1130 * enmFlush).
1131 * @param enmFlush Type of flush.
1132 */
1133static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1134{
1135 AssertPtr(pVM);
1136 Assert(pVM->hm.s.fNestedPaging);
1137
1138 uint64_t descriptor[2];
1139 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1140 descriptor[0] = 0;
1141 else
1142 {
1143 Assert(pVCpu);
1144 descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1145 }
1146 descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1147
1148 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1149 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1150 rc));
1151 if ( RT_SUCCESS(rc)
1152 && pVCpu)
1153 {
1154 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1155 }
1156}
1157
1158
1159/**
1160 * Flushes the TLB using VPID.
1161 *
1162 * @returns VBox status code.
1163 * @param pVM Pointer to the VM.
1164 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1165 * enmFlush).
1166 * @param enmFlush Type of flush.
1167 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1168 * on @a enmFlush).
1169 */
1170static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1171{
1172 AssertPtr(pVM);
1173 Assert(pVM->hm.s.vmx.fVpid);
1174
1175 uint64_t descriptor[2];
1176 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1177 {
1178 descriptor[0] = 0;
1179 descriptor[1] = 0;
1180 }
1181 else
1182 {
1183 AssertPtr(pVCpu);
1184 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1185 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1186 descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1187 descriptor[1] = GCPtr;
1188 }
1189
1190 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1191 AssertMsg(rc == VINF_SUCCESS,
1192 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1193 if ( RT_SUCCESS(rc)
1194 && pVCpu)
1195 {
1196 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1197 }
1198}
1199
1200
1201/**
1202 * Invalidates a guest page by guest virtual address. Only relevant for
1203 * EPT/VPID, otherwise there is nothing really to invalidate.
1204 *
1205 * @returns VBox status code.
1206 * @param pVM Pointer to the VM.
1207 * @param pVCpu Pointer to the VMCPU.
1208 * @param GCVirt Guest virtual address of the page to invalidate.
1209 */
1210VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1211{
1212 AssertPtr(pVM);
1213 AssertPtr(pVCpu);
1214 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1215
1216 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1217 if (!fFlushPending)
1218 {
1219 /*
1220 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1221 * See @bugref{6043} and @bugref{6177}.
1222 *
1223 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1224 * function maybe called in a loop with individual addresses.
1225 */
1226 if (pVM->hm.s.vmx.fVpid)
1227 {
1228 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1229 {
1230 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1231 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1232 }
1233 else
1234 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1235 }
1236 else if (pVM->hm.s.fNestedPaging)
1237 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1238 }
1239
1240 return VINF_SUCCESS;
1241}
1242
1243
1244/**
1245 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1246 * otherwise there is nothing really to invalidate.
1247 *
1248 * @returns VBox status code.
1249 * @param pVM Pointer to the VM.
1250 * @param pVCpu Pointer to the VMCPU.
1251 * @param GCPhys Guest physical address of the page to invalidate.
1252 */
1253VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1254{
1255 LogFlowFunc(("%RGp\n", GCPhys));
1256
1257 /*
1258 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1259 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1260 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1261 */
1262 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1263 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1264 return VINF_SUCCESS;
1265}
1266
1267
1268/**
1269 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1270 * case where neither EPT nor VPID is supported by the CPU.
1271 *
1272 * @param pVM Pointer to the VM.
1273 * @param pVCpu Pointer to the VMCPU.
1274 *
1275 * @remarks Called with interrupts disabled.
1276 */
1277static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1278{
1279 NOREF(pVM);
1280 AssertPtr(pVCpu);
1281 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1282 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1283
1284 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1285 AssertPtr(pCpu);
1286
1287 pVCpu->hm.s.TlbShootdown.cPages = 0;
1288 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1289 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1290 pVCpu->hm.s.fForceTLBFlush = false;
1291 return;
1292}
1293
1294
1295/**
1296 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1297 *
1298 * @param pVM Pointer to the VM.
1299 * @param pVCpu Pointer to the VMCPU.
1300 * @remarks All references to "ASID" in this function pertains to "VPID" in
1301 * Intel's nomenclature. The reason is, to avoid confusion in compare
1302 * statements since the host-CPU copies are named "ASID".
1303 *
1304 * @remarks Called with interrupts disabled.
1305 */
1306static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1307{
1308#ifdef VBOX_WITH_STATISTICS
1309 bool fTlbFlushed = false;
1310# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1311# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1312 if (!fTlbFlushed) \
1313 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1314 } while (0)
1315#else
1316# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1317# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1318#endif
1319
1320 AssertPtr(pVM);
1321 AssertPtr(pVCpu);
1322 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1323 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1324 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1325
1326 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1327 AssertPtr(pCpu);
1328
1329 /*
1330 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1331 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1332 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1333 */
1334 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1335 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1336 {
1337 ++pCpu->uCurrentAsid;
1338 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1339 {
1340 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1341 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1342 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1343 }
1344
1345 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1346 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1347 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1348
1349 /*
1350 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1351 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1352 */
1353 hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1354 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1355 HMVMX_SET_TAGGED_TLB_FLUSHED();
1356 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1357 }
1358
1359 /* Check for explicit TLB shootdowns. */
1360 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1361 {
1362 /*
1363 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1364 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1365 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1366 * but not guest-physical mappings.
1367 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1368 */
1369 hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1370 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1371 HMVMX_SET_TAGGED_TLB_FLUSHED();
1372 }
1373
1374 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1375 * not be executed. See hmQueueInvlPage() where it is commented
1376 * out. Support individual entry flushing someday. */
1377 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1378 {
1379 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1380
1381 /*
1382 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1383 * as supported by the CPU.
1384 */
1385 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1386 {
1387 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1388 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1389 }
1390 else
1391 hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1392
1393 HMVMX_SET_TAGGED_TLB_FLUSHED();
1394 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1395 }
1396
1397 pVCpu->hm.s.TlbShootdown.cPages = 0;
1398 pVCpu->hm.s.fForceTLBFlush = false;
1399
1400 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1401
1402 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1403 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1404 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1405 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1406 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1407 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1408 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1409 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1410
1411 /* Update VMCS with the VPID. */
1412 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1413 AssertRC(rc);
1414
1415#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1416}
1417
1418
1419/**
1420 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1421 *
1422 * @returns VBox status code.
1423 * @param pVM Pointer to the VM.
1424 * @param pVCpu Pointer to the VMCPU.
1425 *
1426 * @remarks Called with interrupts disabled.
1427 */
1428static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1429{
1430 AssertPtr(pVM);
1431 AssertPtr(pVCpu);
1432 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1433 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1434
1435 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1436 AssertPtr(pCpu);
1437
1438 /*
1439 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1440 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1441 */
1442 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1443 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1444 {
1445 pVCpu->hm.s.fForceTLBFlush = true;
1446 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1447 }
1448
1449 /* Check for explicit TLB shootdown flushes. */
1450 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1451 {
1452 pVCpu->hm.s.fForceTLBFlush = true;
1453 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1454 }
1455
1456 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1457 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1458
1459 if (pVCpu->hm.s.fForceTLBFlush)
1460 {
1461 hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1462 pVCpu->hm.s.fForceTLBFlush = false;
1463 }
1464 else
1465 {
1466 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1467 * not be executed. See hmQueueInvlPage() where it is commented
1468 * out. Support individual entry flushing someday. */
1469 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1470 {
1471 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1472 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1473 hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1474 }
1475 else
1476 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1477 }
1478
1479 pVCpu->hm.s.TlbShootdown.cPages = 0;
1480 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1481}
1482
1483
1484/**
1485 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1486 *
1487 * @returns VBox status code.
1488 * @param pVM Pointer to the VM.
1489 * @param pVCpu Pointer to the VMCPU.
1490 *
1491 * @remarks Called with interrupts disabled.
1492 */
1493static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1494{
1495 AssertPtr(pVM);
1496 AssertPtr(pVCpu);
1497 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1498 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1499
1500 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1501
1502 /*
1503 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1504 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1505 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1506 */
1507 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1508 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1509 {
1510 pVCpu->hm.s.fForceTLBFlush = true;
1511 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1512 }
1513
1514 /* Check for explicit TLB shootdown flushes. */
1515 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1516 {
1517 /*
1518 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1519 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1520 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1521 */
1522 pVCpu->hm.s.fForceTLBFlush = true;
1523 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1524 }
1525
1526 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1527 if (pVCpu->hm.s.fForceTLBFlush)
1528 {
1529 ++pCpu->uCurrentAsid;
1530 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1531 {
1532 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1533 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1534 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1535 }
1536
1537 pVCpu->hm.s.fForceTLBFlush = false;
1538 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1539 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1540 if (pCpu->fFlushAsidBeforeUse)
1541 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1542 }
1543 else
1544 {
1545 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1546 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1547 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1548 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1549
1550 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1551 * not be executed. See hmQueueInvlPage() where it is commented
1552 * out. Support individual entry flushing someday. */
1553 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1554 {
1555 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1556 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1557 {
1558 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1559 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1560 }
1561 else
1562 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1563 }
1564 else
1565 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1566 }
1567
1568 pVCpu->hm.s.TlbShootdown.cPages = 0;
1569 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1570
1571 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1572 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1573 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1574 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1575 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1576 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1577
1578 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1579 AssertRC(rc);
1580}
1581
1582
1583/**
1584 * Flushes the guest TLB entry based on CPU capabilities.
1585 *
1586 * @param pVCpu Pointer to the VMCPU.
1587 */
1588DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1589{
1590 PVM pVM = pVCpu->CTX_SUFF(pVM);
1591 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1592 {
1593 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1594 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1595 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1596 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1597 default:
1598 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1599 break;
1600 }
1601}
1602
1603
1604/**
1605 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1606 * TLB entries from the host TLB before VM-entry.
1607 *
1608 * @returns VBox status code.
1609 * @param pVM Pointer to the VM.
1610 */
1611static int hmR0VmxSetupTaggedTlb(PVM pVM)
1612{
1613 /*
1614 * Determine optimal flush type for Nested Paging.
1615 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1616 * guest execution (see hmR3InitFinalizeR0()).
1617 */
1618 if (pVM->hm.s.fNestedPaging)
1619 {
1620 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1621 {
1622 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1623 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1624 else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1625 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1626 else
1627 {
1628 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1629 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1630 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1631 }
1632
1633 /* Make sure the write-back cacheable memory type for EPT is supported. */
1634 if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1635 {
1636 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1637 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1638 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1639 }
1640 }
1641 else
1642 {
1643 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1644 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1645 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1646 }
1647 }
1648
1649 /*
1650 * Determine optimal flush type for VPID.
1651 */
1652 if (pVM->hm.s.vmx.fVpid)
1653 {
1654 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1655 {
1656 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1657 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1658 else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1659 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1660 else
1661 {
1662 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1663 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1664 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1665 if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1666 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1667 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1668 pVM->hm.s.vmx.fVpid = false;
1669 }
1670 }
1671 else
1672 {
1673 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1674 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1675 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1676 pVM->hm.s.vmx.fVpid = false;
1677 }
1678 }
1679
1680 /*
1681 * Setup the handler for flushing tagged-TLBs.
1682 */
1683 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1684 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1685 else if (pVM->hm.s.fNestedPaging)
1686 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1687 else if (pVM->hm.s.vmx.fVpid)
1688 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1689 else
1690 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1691 return VINF_SUCCESS;
1692}
1693
1694
1695/**
1696 * Sets up pin-based VM-execution controls in the VMCS.
1697 *
1698 * @returns VBox status code.
1699 * @param pVM Pointer to the VM.
1700 * @param pVCpu Pointer to the VMCPU.
1701 */
1702static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1703{
1704 AssertPtr(pVM);
1705 AssertPtr(pVCpu);
1706
1707 uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1708 uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1709
1710 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1711 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1712 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1713
1714 /* Enable the VMX preemption timer. */
1715 if (pVM->hm.s.vmx.fUsePreemptTimer)
1716 {
1717 Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1718 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1719 }
1720
1721 if ((val & zap) != val)
1722 {
1723 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1724 pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1725 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1726 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1727 }
1728
1729 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1730 AssertRCReturn(rc, rc);
1731
1732 /* Update VCPU with the currently set pin-based VM-execution controls. */
1733 pVCpu->hm.s.vmx.u32PinCtls = val;
1734 return rc;
1735}
1736
1737
1738/**
1739 * Sets up processor-based VM-execution controls in the VMCS.
1740 *
1741 * @returns VBox status code.
1742 * @param pVM Pointer to the VM.
1743 * @param pVMCPU Pointer to the VMCPU.
1744 */
1745static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1746{
1747 AssertPtr(pVM);
1748 AssertPtr(pVCpu);
1749
1750 int rc = VERR_INTERNAL_ERROR_5;
1751 uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1752 uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1753
1754 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1755 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1756 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1757 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1758 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1759 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1760 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1761
1762 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1763 if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1764 || (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1765 {
1766 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1767 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1768 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1769 }
1770
1771 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1772 if (!pVM->hm.s.fNestedPaging)
1773 {
1774 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1775 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1776 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1777 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1778 }
1779
1780 /* Use TPR shadowing if supported by the CPU. */
1781 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1782 {
1783 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1784 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1785 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1786 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1787 AssertRCReturn(rc, rc);
1788
1789 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1790 /* CR8 writes causes a VM-exit based on TPR threshold. */
1791 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1792 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1793 }
1794 else
1795 {
1796 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1797 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1798 }
1799
1800 /* Use MSR-bitmaps if supported by the CPU. */
1801 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1802 {
1803 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1804
1805 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1806 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1807 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1808 AssertRCReturn(rc, rc);
1809
1810 /*
1811 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1812 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1813 */
1814 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1815 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1816 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1817 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1818 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1819 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1820 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1821 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1822 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1823 }
1824
1825 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1826 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1827 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1828
1829 if ((val & zap) != val)
1830 {
1831 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1832 pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1833 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1834 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1835 }
1836
1837 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1838 AssertRCReturn(rc, rc);
1839
1840 /* Update VCPU with the currently set processor-based VM-execution controls. */
1841 pVCpu->hm.s.vmx.u32ProcCtls = val;
1842
1843 /*
1844 * Secondary processor-based VM-execution controls.
1845 */
1846 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1847 {
1848 val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1849 zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1850
1851 if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1852 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1853
1854 if (pVM->hm.s.fNestedPaging)
1855 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1856 else
1857 {
1858 /*
1859 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1860 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1861 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1862 */
1863 if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1864 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1865 }
1866
1867 if (pVM->hm.s.vmx.fVpid)
1868 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1869
1870 if (pVM->hm.s.vmx.fUnrestrictedGuest)
1871 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1872
1873 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1874 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1875 * done dynamically. */
1876 if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1877 {
1878 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1879 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1880 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1881 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1882 AssertRCReturn(rc, rc);
1883 }
1884
1885 if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1886 {
1887 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1888 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1889 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1890 }
1891
1892 if ((val & zap) != val)
1893 {
1894 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1895 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1896 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1897 }
1898
1899 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1900 AssertRCReturn(rc, rc);
1901
1902 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1903 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1904 }
1905 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1906 {
1907 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1908 "available\n"));
1909 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1910 }
1911
1912 return VINF_SUCCESS;
1913}
1914
1915
1916/**
1917 * Sets up miscellaneous (everything other than Pin & Processor-based
1918 * VM-execution) control fields in the VMCS.
1919 *
1920 * @returns VBox status code.
1921 * @param pVM Pointer to the VM.
1922 * @param pVCpu Pointer to the VMCPU.
1923 */
1924static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1925{
1926 AssertPtr(pVM);
1927 AssertPtr(pVCpu);
1928
1929 int rc = VERR_GENERAL_FAILURE;
1930
1931 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1932#if 0
1933 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1934 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1935 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1936
1937 /*
1938 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1939 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1940 * We thus use the exception bitmap to control it rather than use both.
1941 */
1942 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1943 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1944
1945 /** @todo Explore possibility of using IO-bitmaps. */
1946 /* All IO & IOIO instructions cause VM-exits. */
1947 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1948 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1949
1950 /* Initialize the MSR-bitmap area. */
1951 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1952 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1953 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1954#endif
1955
1956#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1957 /* Setup MSR autoloading/storing. */
1958 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1959 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1960 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1961 AssertRCReturn(rc, rc);
1962 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1963 AssertRCReturn(rc, rc);
1964
1965 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1966 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1967 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1968 AssertRCReturn(rc, rc);
1969#endif
1970
1971 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1972 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1973 AssertRCReturn(rc, rc);
1974
1975 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1976#if 0
1977 /* Setup debug controls */
1978 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1979 AssertRCReturn(rc, rc);
1980 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1981 AssertRCReturn(rc, rc);
1982#endif
1983
1984 return rc;
1985}
1986
1987
1988/**
1989 * Sets up the initial exception bitmap in the VMCS based on static conditions
1990 * (i.e. conditions that cannot ever change at runtime).
1991 *
1992 * @returns VBox status code.
1993 * @param pVM Pointer to the VM.
1994 * @param pVCpu Pointer to the VMCPU.
1995 */
1996static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1997{
1998 AssertPtr(pVM);
1999 AssertPtr(pVCpu);
2000
2001 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2002
2003 uint32_t u32XcptBitmap = 0;
2004
2005 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2006 if (!pVM->hm.s.fNestedPaging)
2007 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2008
2009 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2010 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2011 AssertRCReturn(rc, rc);
2012 return rc;
2013}
2014
2015
2016/**
2017 * Sets up the initial guest-state mask. The guest-state mask is consulted
2018 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2019 * for the nested virtualization case (as it would cause a VM-exit).
2020 *
2021 * @param pVCpu Pointer to the VMCPU.
2022 */
2023static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2024{
2025 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2026 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2027 return VINF_SUCCESS;
2028}
2029
2030
2031/**
2032 * Does per-VM VT-x initialization.
2033 *
2034 * @returns VBox status code.
2035 * @param pVM Pointer to the VM.
2036 */
2037VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2038{
2039 LogFlowFunc(("pVM=%p\n", pVM));
2040
2041 int rc = hmR0VmxStructsAlloc(pVM);
2042 if (RT_FAILURE(rc))
2043 {
2044 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2045 return rc;
2046 }
2047
2048 return VINF_SUCCESS;
2049}
2050
2051
2052/**
2053 * Does per-VM VT-x termination.
2054 *
2055 * @returns VBox status code.
2056 * @param pVM Pointer to the VM.
2057 */
2058VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2059{
2060 LogFlowFunc(("pVM=%p\n", pVM));
2061
2062#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2063 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2064 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2065#endif
2066 hmR0VmxStructsFree(pVM);
2067 return VINF_SUCCESS;
2068}
2069
2070
2071/**
2072 * Sets up the VM for execution under VT-x.
2073 * This function is only called once per-VM during initialization.
2074 *
2075 * @returns VBox status code.
2076 * @param pVM Pointer to the VM.
2077 */
2078VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2079{
2080 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2081 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2082
2083 LogFlowFunc(("pVM=%p\n", pVM));
2084
2085 /*
2086 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2087 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2088 */
2089 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2090 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2091 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2092 || !pVM->hm.s.vmx.pRealModeTSS))
2093 {
2094 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2095 return VERR_INTERNAL_ERROR;
2096 }
2097
2098#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2099 /*
2100 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2101 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2102 */
2103 if ( pVM->hm.s.fAllow64BitGuests
2104 && !HMVMX_IS_64BIT_HOST_MODE())
2105 {
2106 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2107 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2108 }
2109#endif
2110
2111 /* Initialize these always, see hmR3InitFinalizeR0().*/
2112 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2113 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2114
2115 /* Setup the tagged-TLB flush handlers. */
2116 int rc = hmR0VmxSetupTaggedTlb(pVM);
2117 if (RT_FAILURE(rc))
2118 {
2119 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2120 return rc;
2121 }
2122
2123 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2124 {
2125 PVMCPU pVCpu = &pVM->aCpus[i];
2126 AssertPtr(pVCpu);
2127 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2128
2129 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2130 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2131
2132 /* Set revision dword at the beginning of the VMCS structure. */
2133 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
2134
2135 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2136 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2137 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2138 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2139
2140 /* Load this VMCS as the current VMCS. */
2141 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2142 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2143 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2144
2145 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2146 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2147 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2148
2149 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2150 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2151 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2152
2153 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2154 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2155 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2156
2157 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2158 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2159 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2160
2161 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2162 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2163 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2164
2165#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2166 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2167 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2168 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2169#endif
2170
2171 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2172 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2173 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2174 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2175
2176 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2177
2178 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2179 }
2180
2181 return VINF_SUCCESS;
2182}
2183
2184
2185/**
2186 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2187 * the VMCS.
2188 *
2189 * @returns VBox status code.
2190 * @param pVM Pointer to the VM.
2191 * @param pVCpu Pointer to the VMCPU.
2192 */
2193DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2194{
2195 RTCCUINTREG uReg = ASMGetCR0();
2196 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2197 AssertRCReturn(rc, rc);
2198
2199#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2200 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2201 if (HMVMX_IS_64BIT_HOST_MODE())
2202 {
2203 uint64_t uRegCR3 = HMR0Get64bitCR3();
2204 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2205 }
2206 else
2207#endif
2208 {
2209 uReg = ASMGetCR3();
2210 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2211 }
2212 AssertRCReturn(rc, rc);
2213
2214 uReg = ASMGetCR4();
2215 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2216 AssertRCReturn(rc, rc);
2217 return rc;
2218}
2219
2220
2221/**
2222 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2223 * the host-state area in the VMCS.
2224 *
2225 * @returns VBox status code.
2226 * @param pVM Pointer to the VM.
2227 * @param pVCpu Pointer to the VMCPU.
2228 */
2229DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2230{
2231 int rc = VERR_INTERNAL_ERROR_5;
2232 RTSEL uSelDS = 0;
2233 RTSEL uSelES = 0;
2234 RTSEL uSelFS = 0;
2235 RTSEL uSelGS = 0;
2236 RTSEL uSelTR = 0;
2237
2238 /*
2239 * Host DS, ES, FS and GS segment registers.
2240 */
2241#if HC_ARCH_BITS == 64
2242 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2243 uSelDS = ASMGetDS();
2244 uSelES = ASMGetES();
2245 uSelFS = ASMGetFS();
2246 uSelGS = ASMGetGS();
2247#endif
2248
2249 /*
2250 * Host CS and SS segment registers.
2251 */
2252 RTSEL uSelCS;
2253 RTSEL uSelSS;
2254#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2255 if (HMVMX_IS_64BIT_HOST_MODE())
2256 {
2257 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2258 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2259 }
2260 else
2261 {
2262 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2263 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2264 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2265 }
2266#else
2267 uSelCS = ASMGetCS();
2268 uSelSS = ASMGetSS();
2269#endif
2270
2271 /*
2272 * Host TR segment register.
2273 */
2274 uSelTR = ASMGetTR();
2275
2276#if HC_ARCH_BITS == 64
2277 /*
2278 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2279 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2280 */
2281 if (uSelDS & (X86_SEL_RPL | X86_SEL_LDT))
2282 {
2283 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_DS;
2284 pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2285 uSelDS = 0;
2286 }
2287 if (uSelES & (X86_SEL_RPL | X86_SEL_LDT))
2288 {
2289 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_ES;
2290 pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2291 uSelES = 0;
2292 }
2293 if (uSelFS & (X86_SEL_RPL | X86_SEL_LDT))
2294 {
2295 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_FS;
2296 pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2297 uSelFS = 0;
2298 }
2299 if (uSelGS & (X86_SEL_RPL | X86_SEL_LDT))
2300 {
2301 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_GS;
2302 pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2303 uSelGS = 0;
2304 }
2305#endif
2306
2307 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2308 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2309 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2310 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2311 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2312 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2313 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2314 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2315 Assert(uSelCS);
2316 Assert(uSelTR);
2317
2318 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2319#if 0
2320 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2321 Assert(uSelSS != 0);
2322#endif
2323
2324 /* Write these host selector fields into the host-state area in the VMCS. */
2325 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2326 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2327#if HC_ARCH_BITS == 64
2328 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2329 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2330 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2331 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2332#endif
2333 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2334
2335 /*
2336 * Host GDTR and IDTR.
2337 */
2338 RTGDTR Gdtr;
2339 RT_ZERO(Gdtr);
2340#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2341 if (HMVMX_IS_64BIT_HOST_MODE())
2342 {
2343 X86XDTR64 Gdtr64;
2344 X86XDTR64 Idtr64;
2345 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2346 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2347 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2348
2349 Gdtr.cbGdt = Gdtr64.cb;
2350 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2351 }
2352 else
2353#endif
2354 {
2355 RTIDTR Idtr;
2356 ASMGetGDTR(&Gdtr);
2357 ASMGetIDTR(&Idtr);
2358 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2359 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2360
2361#if HC_ARCH_BITS == 64
2362 /*
2363 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2364 * maximum limit (0xffff) on every VM-exit.
2365 */
2366 if (Gdtr.cbGdt != 0xffff)
2367 {
2368 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2369 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2370 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2371 }
2372
2373 /*
2374 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2375 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2376 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2377 */
2378 if (Idtr.cbIdt < 0x0fff)
2379 {
2380 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2381 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2382 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2383 }
2384#endif
2385 }
2386
2387 /*
2388 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2389 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2390 */
2391 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2392 {
2393 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2394 return VERR_VMX_INVALID_HOST_STATE;
2395 }
2396
2397 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2398#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2399 if (HMVMX_IS_64BIT_HOST_MODE())
2400 {
2401 /* We need the 64-bit TR base for hybrid darwin. */
2402 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2403 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2404 }
2405 else
2406#endif
2407 {
2408 uintptr_t uTRBase;
2409#if HC_ARCH_BITS == 64
2410 uTRBase = X86DESC64_BASE(pDesc);
2411
2412 /*
2413 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2414 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2415 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2416 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2417 *
2418 * [1] See Intel spec. 3.5 "System Descriptor Types".
2419 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2420 */
2421 Assert(pDesc->System.u4Type == 11);
2422 if ( pDesc->System.u16LimitLow != 0x67
2423 || pDesc->System.u4LimitHigh)
2424 {
2425 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2426 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2427
2428 /* Store the GDTR here as we need it while restoring TR. */
2429 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2430 }
2431#else
2432 uTRBase = X86DESC_BASE(pDesc);
2433#endif
2434 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2435 }
2436 AssertRCReturn(rc, rc);
2437
2438 /*
2439 * Host FS base and GS base.
2440 */
2441#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2442 if (HMVMX_IS_64BIT_HOST_MODE())
2443 {
2444 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2445 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2446 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2447 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2448
2449# if HC_ARCH_BITS == 64
2450 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2451 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2452 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2453 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2454 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2455# endif
2456 }
2457#endif
2458 return rc;
2459}
2460
2461
2462/**
2463 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2464 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2465 * the host after every successful VM exit.
2466 *
2467 * @returns VBox status code.
2468 * @param pVM Pointer to the VM.
2469 * @param pVCpu Pointer to the VMCPU.
2470 */
2471DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2472{
2473 AssertPtr(pVCpu);
2474 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2475
2476 int rc = VINF_SUCCESS;
2477#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2478 PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2479 uint32_t cHostMsrs = 0;
2480 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2481
2482 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2483 {
2484 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2485
2486# if HC_ARCH_BITS == 64
2487 /* Paranoia. 64-bit code requires these bits to be set always. */
2488 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2489
2490 /*
2491 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2492 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2493 * some reason (e.g. allow transparent reads) we would activate the code below.
2494 */
2495# if 0
2496 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2497 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2498 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2499 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2500 if (CPUMIsGuestInLongMode(pVCpu))
2501 {
2502 uint64_t u64GuestEfer;
2503 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2504 AssertRC(rc);
2505
2506 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2507 {
2508 pHostMsr->u32IndexMSR = MSR_K6_EFER;
2509 pHostMsr->u32Reserved = 0;
2510 pHostMsr->u64Value = u64HostEfer;
2511 pHostMsr++; cHostMsrs++;
2512 }
2513 }
2514# endif
2515# else /* HC_ARCH_BITS != 64 */
2516 pHostMsr->u32IndexMSR = MSR_K6_EFER;
2517 pHostMsr->u32Reserved = 0;
2518# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2519 if (CPUMIsGuestInLongMode(pVCpu))
2520 {
2521 /* Must match the EFER value in our 64 bits switcher. */
2522 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2523 }
2524 else
2525# endif
2526 pHostMsr->u64Value = u64HostEfer;
2527 pHostMsr++; cHostMsrs++;
2528# endif /* HC_ARCH_BITS == 64 */
2529 }
2530
2531# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2532 if (HMVMX_IS_64BIT_HOST_MODE())
2533 {
2534 pHostMsr->u32IndexMSR = MSR_K6_STAR;
2535 pHostMsr->u32Reserved = 0;
2536 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2537 pHostMsr++; cHostMsrs++;
2538 pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2539 pHostMsr->u32Reserved = 0;
2540 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2541 pHostMsr++; cHostMsrs++;
2542 pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2543 pHostMsr->u32Reserved = 0;
2544 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2545 pHostMsr++; cHostMsrs++;
2546 pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2547 pHostMsr->u32Reserved = 0;
2548 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2549 pHostMsr++; cHostMsrs++;
2550 }
2551# endif
2552
2553 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2554 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2555 {
2556 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2557 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2558 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2559 }
2560
2561 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2562#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2563
2564 /*
2565 * Host Sysenter MSRs.
2566 */
2567 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2568 AssertRCReturn(rc, rc);
2569#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2570 if (HMVMX_IS_64BIT_HOST_MODE())
2571 {
2572 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2573 AssertRCReturn(rc, rc);
2574 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2575 }
2576 else
2577 {
2578 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2579 AssertRCReturn(rc, rc);
2580 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2581 }
2582#elif HC_ARCH_BITS == 32
2583 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2584 AssertRCReturn(rc, rc);
2585 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2586#else
2587 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2588 AssertRCReturn(rc, rc);
2589 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2590#endif
2591 AssertRCReturn(rc, rc);
2592
2593 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2594 * hmR0VmxSetupExitCtls() !! */
2595 return rc;
2596}
2597
2598
2599/**
2600 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2601 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2602 * controls".
2603 *
2604 * @returns VBox status code.
2605 * @param pVCpu Pointer to the VMCPU.
2606 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2607 * out-of-sync. Make sure to update the required fields
2608 * before using them.
2609 *
2610 * @remarks No-long-jump zone!!!
2611 */
2612DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2613{
2614 int rc = VINF_SUCCESS;
2615 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2616 {
2617 PVM pVM = pVCpu->CTX_SUFF(pVM);
2618 uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2619 uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2620
2621 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2622 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2623
2624 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2625 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2626 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2627 else
2628 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2629
2630 /*
2631 * The following should not be set (since we're not in SMM mode):
2632 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2633 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2634 */
2635
2636 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2637 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2638 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2639
2640 if ((val & zap) != val)
2641 {
2642 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2643 pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2644 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2645 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2646 }
2647
2648 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2649 AssertRCReturn(rc, rc);
2650
2651 /* Update VCPU with the currently set VM-exit controls. */
2652 pVCpu->hm.s.vmx.u32EntryCtls = val;
2653 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2654 }
2655 return rc;
2656}
2657
2658
2659/**
2660 * Sets up the VM-exit controls in the VMCS.
2661 *
2662 * @returns VBox status code.
2663 * @param pVM Pointer to the VM.
2664 * @param pVCpu Pointer to the VMCPU.
2665 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2666 * out-of-sync. Make sure to update the required fields
2667 * before using them.
2668 *
2669 * @remarks requires EFER.
2670 */
2671DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2672{
2673 int rc = VINF_SUCCESS;
2674 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2675 {
2676 PVM pVM = pVCpu->CTX_SUFF(pVM);
2677 uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2678 uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2679
2680 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2681 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2682
2683 /*
2684 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2685 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2686 */
2687#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2688 if (HMVMX_IS_64BIT_HOST_MODE())
2689 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2690 else
2691 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2692#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2693 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2694 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2695 else
2696 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2697#endif
2698
2699 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2700 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2701
2702 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2703 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2704 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2705 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2706 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2707
2708 if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2709 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2710
2711 if ((val & zap) != val)
2712 {
2713 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2714 pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2715 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2716 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2717 }
2718
2719 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2720 AssertRCReturn(rc, rc);
2721
2722 /* Update VCPU with the currently set VM-exit controls. */
2723 pVCpu->hm.s.vmx.u32ExitCtls = val;
2724 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2725 }
2726 return rc;
2727}
2728
2729
2730/**
2731 * Loads the guest APIC and related state.
2732 *
2733 * @returns VBox status code.
2734 * @param pVM Pointer to the VM.
2735 * @param pVCpu Pointer to the VMCPU.
2736 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2737 * out-of-sync. Make sure to update the required fields
2738 * before using them.
2739 */
2740DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2741{
2742 int rc = VINF_SUCCESS;
2743 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2744 {
2745 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2746 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2747 {
2748 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2749
2750 bool fPendingIntr = false;
2751 uint8_t u8Tpr = 0;
2752 uint8_t u8PendingIntr = 0;
2753 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2754 AssertRCReturn(rc, rc);
2755
2756 /*
2757 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2758 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2759 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2760 * the interrupt when we VM-exit for other reasons.
2761 */
2762 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2763 uint32_t u32TprThreshold = 0;
2764 if (fPendingIntr)
2765 {
2766 /* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2767 const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2768 const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2769 if (u8PendingPriority <= u8TprPriority)
2770 u32TprThreshold = u8PendingPriority;
2771 else
2772 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2773 }
2774 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2775
2776 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2777 AssertRCReturn(rc, rc);
2778 }
2779
2780 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2781 }
2782 return rc;
2783}
2784
2785
2786/**
2787 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2788 *
2789 * @returns Guest's interruptibility-state.
2790 * @param pVCpu Pointer to the VMCPU.
2791 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2792 * out-of-sync. Make sure to update the required fields
2793 * before using them.
2794 *
2795 * @remarks No-long-jump zone!!!
2796 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2797 */
2798DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2799{
2800 /*
2801 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2802 * inhibit interrupts or clear any existing interrupt-inhibition.
2803 */
2804 uint32_t uIntrState = 0;
2805 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2806 {
2807 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2808 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
2809 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2810 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2811 {
2812 /*
2813 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2814 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2815 */
2816 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2817 }
2818 else if (pMixedCtx->eflags.Bits.u1IF)
2819 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2820 else
2821 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2822 }
2823 return uIntrState;
2824}
2825
2826
2827/**
2828 * Loads the guest's interruptibility-state into the guest-state area in the
2829 * VMCS.
2830 *
2831 * @returns VBox status code.
2832 * @param pVCpu Pointer to the VMCPU.
2833 * @param uIntrState The interruptibility-state to set.
2834 */
2835static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2836{
2837 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2838 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2839 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2840 AssertRCReturn(rc, rc);
2841 return rc;
2842}
2843
2844
2845/**
2846 * Loads the guest's RIP into the guest-state area in the VMCS.
2847 *
2848 * @returns VBox status code.
2849 * @param pVCpu Pointer to the VMCPU.
2850 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2851 * out-of-sync. Make sure to update the required fields
2852 * before using them.
2853 *
2854 * @remarks No-long-jump zone!!!
2855 */
2856static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2857{
2858 int rc = VINF_SUCCESS;
2859 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2860 {
2861 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2862 AssertRCReturn(rc, rc);
2863 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2864 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#x\n", pMixedCtx->rip, pVCpu->hm.s.fContextUseFlags));
2865 }
2866 return rc;
2867}
2868
2869
2870/**
2871 * Loads the guest's RSP into the guest-state area in the VMCS.
2872 *
2873 * @returns VBox status code.
2874 * @param pVCpu Pointer to the VMCPU.
2875 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2876 * out-of-sync. Make sure to update the required fields
2877 * before using them.
2878 *
2879 * @remarks No-long-jump zone!!!
2880 */
2881static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2882{
2883 int rc = VINF_SUCCESS;
2884 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2885 {
2886 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2887 AssertRCReturn(rc, rc);
2888 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2889 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2890 }
2891 return rc;
2892}
2893
2894
2895/**
2896 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
2897 *
2898 * @returns VBox status code.
2899 * @param pVCpu Pointer to the VMCPU.
2900 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2901 * out-of-sync. Make sure to update the required fields
2902 * before using them.
2903 *
2904 * @remarks No-long-jump zone!!!
2905 */
2906static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2907{
2908 int rc = VINF_SUCCESS;
2909 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2910 {
2911 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2912 Let us assert it as such and use 32-bit VMWRITE. */
2913 Assert(!(pMixedCtx->rflags.u64 >> 32));
2914 X86EFLAGS Eflags = pMixedCtx->eflags;
2915 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2916 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2917
2918 /*
2919 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2920 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2921 */
2922 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2923 {
2924 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2925 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2926 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2927 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2928 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2929 }
2930
2931 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2932 AssertRCReturn(rc, rc);
2933
2934 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2935 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2936 }
2937 return rc;
2938}
2939
2940
2941/**
2942 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2943 *
2944 * @returns VBox status code.
2945 * @param pVCpu Pointer to the VMCPU.
2946 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2947 * out-of-sync. Make sure to update the required fields
2948 * before using them.
2949 *
2950 * @remarks No-long-jump zone!!!
2951 */
2952DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2953{
2954 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2955 AssertRCReturn(rc, rc);
2956 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2957 AssertRCReturn(rc, rc);
2958 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2959 AssertRCReturn(rc, rc);
2960 return rc;
2961}
2962
2963
2964/**
2965 * Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2966 * in the VMCS.
2967 *
2968 * @returns VBox status code.
2969 * @param pVM Pointer to the VM.
2970 * @param pVCpu Pointer to the VMCPU.
2971 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2972 * out-of-sync. Make sure to update the required fields
2973 * before using them.
2974 *
2975 * @remarks No-long-jump zone!!!
2976 */
2977static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2978{
2979 int rc = VINF_SUCCESS;
2980 PVM pVM = pVCpu->CTX_SUFF(pVM);
2981
2982 /*
2983 * Guest CR0.
2984 * Guest FPU.
2985 */
2986 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2987 {
2988 Assert(!(pCtx->cr0 >> 32));
2989 uint32_t u32GuestCR0 = pCtx->cr0;
2990
2991 /* The guest's view (read access) of its CR0 is unblemished. */
2992 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2993 AssertRCReturn(rc, rc);
2994 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2995
2996 /* Setup VT-x's view of the guest CR0. */
2997 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2998 if (pVM->hm.s.fNestedPaging)
2999 {
3000 if (CPUMIsGuestPagingEnabledEx(pCtx))
3001 {
3002 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
3003 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3004 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3005 }
3006 else
3007 {
3008 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3009 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3010 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3011 }
3012
3013 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3014 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3015 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3016
3017 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3018 AssertRCReturn(rc, rc);
3019 }
3020 else
3021 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3022
3023 /*
3024 * Guest FPU bits.
3025 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3026 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3027 */
3028 u32GuestCR0 |= X86_CR0_NE;
3029 bool fInterceptNM = false;
3030 if (CPUMIsGuestFPUStateActive(pVCpu))
3031 {
3032 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3033 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3034 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3035 }
3036 else
3037 {
3038 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3039 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3040 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3041 }
3042
3043 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3044 bool fInterceptMF = false;
3045 if (!(pCtx->cr0 & X86_CR0_NE))
3046 fInterceptMF = true;
3047
3048 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3049 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3050 {
3051 Assert(PDMVmmDevHeapIsEnabled(pVM));
3052 Assert(pVM->hm.s.vmx.pRealModeTSS);
3053 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3054 fInterceptNM = true;
3055 fInterceptMF = true;
3056 }
3057 else
3058 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3059
3060 if (fInterceptNM)
3061 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3062 else
3063 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3064
3065 if (fInterceptMF)
3066 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3067 else
3068 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3069
3070 /* Additional intercepts for debugging, define these yourself explicitly. */
3071#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3072 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3073 | RT_BIT(X86_XCPT_BP)
3074 | RT_BIT(X86_XCPT_DB)
3075 | RT_BIT(X86_XCPT_DE)
3076 | RT_BIT(X86_XCPT_NM)
3077 | RT_BIT(X86_XCPT_UD)
3078 | RT_BIT(X86_XCPT_NP)
3079 | RT_BIT(X86_XCPT_SS)
3080 | RT_BIT(X86_XCPT_GP)
3081 | RT_BIT(X86_XCPT_PF)
3082 | RT_BIT(X86_XCPT_MF)
3083 ;
3084#elif defined(HMVMX_ALWAYS_TRAP_PF)
3085 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3086#endif
3087
3088 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3089
3090 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3091 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3092 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 | pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
3093 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3094 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3095 else
3096 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3097
3098 u32GuestCR0 |= uSetCR0;
3099 u32GuestCR0 &= uZapCR0;
3100 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3101
3102 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3103 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3104 AssertRCReturn(rc, rc);
3105 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3106 AssertRCReturn(rc, rc);
3107 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3108
3109 /*
3110 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3111 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3112 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3113 */
3114 uint32_t u32CR0Mask = 0;
3115 u32CR0Mask = X86_CR0_PE
3116 | X86_CR0_NE
3117 | X86_CR0_WP
3118 | X86_CR0_PG
3119 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3120 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3121 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3122 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3123 u32CR0Mask &= ~X86_CR0_PE;
3124 if (pVM->hm.s.fNestedPaging)
3125 u32CR0Mask &= ~X86_CR0_WP;
3126
3127 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3128 if (fInterceptNM)
3129 u32CR0Mask |= (X86_CR0_TS | X86_CR0_MP);
3130 else
3131 u32CR0Mask &= ~(X86_CR0_TS | X86_CR0_MP);
3132
3133 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3134 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3135 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3136 AssertRCReturn(rc, rc);
3137
3138 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3139 }
3140
3141 /*
3142 * Guest CR2.
3143 * It's always loaded in the assembler code. Nothing to do here.
3144 */
3145
3146 /*
3147 * Guest CR3.
3148 */
3149 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3150 {
3151 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3152 if (pVM->hm.s.fNestedPaging)
3153 {
3154 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3155
3156 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3157 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3158 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3159 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3160
3161 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3162 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3163 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3164
3165 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3166 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3167 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3168 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3169
3170 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3171 AssertRCReturn(rc, rc);
3172 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3173
3174 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3175 || CPUMIsGuestPagingEnabledEx(pCtx))
3176 {
3177 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3178 if (CPUMIsGuestInPAEModeEx(pCtx))
3179 {
3180 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3181 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3182 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3183 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3184 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3185 }
3186
3187 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3188 have Unrestricted Execution to handle the guest when it's not using paging. */
3189 GCPhysGuestCR3 = pCtx->cr3;
3190 }
3191 else
3192 {
3193 /*
3194 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3195 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3196 * EPT takes care of translating it to host-physical addresses.
3197 */
3198 RTGCPHYS GCPhys;
3199 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3200 Assert(PDMVmmDevHeapIsEnabled(pVM));
3201
3202 /* We obtain it here every time as the guest could have relocated this PCI region. */
3203 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3204 AssertRCReturn(rc, rc);
3205
3206 GCPhysGuestCR3 = GCPhys;
3207 }
3208
3209 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3210 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3211 }
3212 else
3213 {
3214 /* Non-nested paging case, just use the hypervisor's CR3. */
3215 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3216
3217 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3218 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3219 }
3220 AssertRCReturn(rc, rc);
3221
3222 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3223 }
3224
3225 /*
3226 * Guest CR4.
3227 */
3228 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3229 {
3230 Assert(!(pCtx->cr4 >> 32));
3231 uint32_t u32GuestCR4 = pCtx->cr4;
3232
3233 /* The guest's view of its CR4 is unblemished. */
3234 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3235 AssertRCReturn(rc, rc);
3236 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3237
3238 /* Setup VT-x's view of the guest CR4. */
3239 /*
3240 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3241 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3242 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3243 */
3244 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3245 {
3246 Assert(pVM->hm.s.vmx.pRealModeTSS);
3247 Assert(PDMVmmDevHeapIsEnabled(pVM));
3248 u32GuestCR4 &= ~X86_CR4_VME;
3249 }
3250
3251 if (pVM->hm.s.fNestedPaging)
3252 {
3253 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3254 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3255 {
3256 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3257 u32GuestCR4 |= X86_CR4_PSE;
3258 /* Our identity mapping is a 32 bits page directory. */
3259 u32GuestCR4 &= ~X86_CR4_PAE;
3260 }
3261 /* else use guest CR4.*/
3262 }
3263 else
3264 {
3265 /*
3266 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3267 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3268 */
3269 switch (pVCpu->hm.s.enmShadowMode)
3270 {
3271 case PGMMODE_REAL: /* Real-mode. */
3272 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3273 case PGMMODE_32_BIT: /* 32-bit paging. */
3274 {
3275 u32GuestCR4 &= ~X86_CR4_PAE;
3276 break;
3277 }
3278
3279 case PGMMODE_PAE: /* PAE paging. */
3280 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3281 {
3282 u32GuestCR4 |= X86_CR4_PAE;
3283 break;
3284 }
3285
3286 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3287 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3288#ifdef VBOX_ENABLE_64_BITS_GUESTS
3289 break;
3290#endif
3291 default:
3292 AssertFailed();
3293 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3294 }
3295 }
3296
3297 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3298 uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3299 uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 | pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3300 u32GuestCR4 |= uSetCR4;
3301 u32GuestCR4 &= uZapCR4;
3302
3303 /* Write VT-x's view of the guest CR4 into the VMCS. */
3304 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3305 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3306 AssertRCReturn(rc, rc);
3307
3308 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3309 uint32_t u32CR4Mask = 0;
3310 u32CR4Mask = X86_CR4_VME
3311 | X86_CR4_PAE
3312 | X86_CR4_PGE
3313 | X86_CR4_PSE
3314 | X86_CR4_VMXE;
3315 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3316 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3317 AssertRCReturn(rc, rc);
3318
3319 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3320 }
3321 return rc;
3322}
3323
3324
3325/**
3326 * Loads the guest debug registers into the guest-state area in the VMCS.
3327 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3328 *
3329 * @returns VBox status code.
3330 * @param pVCpu Pointer to the VMCPU.
3331 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3332 * out-of-sync. Make sure to update the required fields
3333 * before using them.
3334 *
3335 * @remarks No-long-jump zone!!!
3336 */
3337static int hmR0VmxLoadGuestDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3338{
3339 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3340 return VINF_SUCCESS;
3341
3342#ifdef VBOX_STRICT
3343 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3344 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3345 {
3346 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3347 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3348 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3349 }
3350#endif
3351
3352 int rc;
3353 PVM pVM = pVCpu->CTX_SUFF(pVM);
3354 bool fInterceptDB = false;
3355 bool fInterceptMovDRx = false;
3356 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3357 {
3358 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3359 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3360 {
3361 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3362 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3363 AssertRCReturn(rc, rc);
3364 Assert(fInterceptDB == false);
3365 }
3366 else
3367 {
3368 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3369 pVCpu->hm.s.fClearTrapFlag = true;
3370 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RFLAGS;
3371 fInterceptDB = true;
3372 }
3373 }
3374
3375 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3376 {
3377 /*
3378 * Use the combined guest and host DRx values found in the hypervisor
3379 * register set because the debugger has breakpoints active or someone
3380 * is single stepping on the host side without a monitor trap flag.
3381 *
3382 * Note! DBGF expects a clean DR6 state before executing guest code.
3383 */
3384 if (!CPUMIsHyperDebugStateActive(pVCpu))
3385 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3386 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3387 Assert(CPUMIsHyperDebugStateActive(pVCpu));
3388
3389 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3390 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3391 AssertRCReturn(rc, rc);
3392
3393 fInterceptDB = true;
3394 fInterceptMovDRx = true;
3395 }
3396 else
3397 {
3398 /*
3399 * If the guest has enabled debug registers, we need to load them prior to
3400 * executing guest code so they'll trigger at the right time.
3401 */
3402 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3403 {
3404 if (!CPUMIsGuestDebugStateActive(pVCpu))
3405 {
3406 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3407 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3408 }
3409 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3410 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3411 }
3412 /*
3413 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3414 * must intercept #DB in order to maintain a correct DR6 guest value.
3415 */
3416 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3417 {
3418 fInterceptMovDRx = true;
3419 fInterceptDB = true;
3420 }
3421
3422 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3423 AssertRCReturn(rc, rc);
3424 }
3425
3426 /*
3427 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3428 */
3429 if (fInterceptDB)
3430 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3431 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3432 {
3433#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3434 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3435#endif
3436 }
3437 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3438 AssertRCReturn(rc, rc);
3439
3440 /*
3441 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3442 */
3443 if (fInterceptMovDRx)
3444 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3445 else
3446 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3447 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3448 AssertRCReturn(rc, rc);
3449
3450 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3451 return VINF_SUCCESS;
3452}
3453
3454
3455#ifdef VBOX_STRICT
3456/**
3457 * Strict function to validate segment registers.
3458 *
3459 * @remarks Requires CR0.
3460 */
3461static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3462{
3463 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3464 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3465 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3466 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3467 && ( !CPUMIsGuestInRealModeEx(pCtx)
3468 && !CPUMIsGuestInV86ModeEx(pCtx)))
3469 {
3470 /* Protected mode checks */
3471 /* CS */
3472 Assert(pCtx->cs.Attr.n.u1Present);
3473 Assert(!(pCtx->cs.Attr.u & 0xf00));
3474 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3475 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3476 || !(pCtx->cs.Attr.n.u1Granularity));
3477 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3478 || (pCtx->cs.Attr.n.u1Granularity));
3479 /* CS cannot be loaded with NULL in protected mode. */
3480 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3481 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3482 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3483 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3484 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3485 else
3486 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3487 /* SS */
3488 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3489 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3490 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3491 if ( !(pCtx->cr0 & X86_CR0_PE)
3492 || pCtx->cs.Attr.n.u4Type == 3)
3493 {
3494 Assert(!pCtx->ss.Attr.n.u2Dpl);
3495 }
3496 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3497 {
3498 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3499 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3500 Assert(pCtx->ss.Attr.n.u1Present);
3501 Assert(!(pCtx->ss.Attr.u & 0xf00));
3502 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3503 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3504 || !(pCtx->ss.Attr.n.u1Granularity));
3505 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3506 || (pCtx->ss.Attr.n.u1Granularity));
3507 }
3508 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3509 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3510 {
3511 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3512 Assert(pCtx->ds.Attr.n.u1Present);
3513 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3514 Assert(!(pCtx->ds.Attr.u & 0xf00));
3515 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3516 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3517 || !(pCtx->ds.Attr.n.u1Granularity));
3518 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3519 || (pCtx->ds.Attr.n.u1Granularity));
3520 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3521 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3522 }
3523 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3524 {
3525 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3526 Assert(pCtx->es.Attr.n.u1Present);
3527 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3528 Assert(!(pCtx->es.Attr.u & 0xf00));
3529 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3530 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3531 || !(pCtx->es.Attr.n.u1Granularity));
3532 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3533 || (pCtx->es.Attr.n.u1Granularity));
3534 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3535 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3536 }
3537 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3538 {
3539 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3540 Assert(pCtx->fs.Attr.n.u1Present);
3541 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3542 Assert(!(pCtx->fs.Attr.u & 0xf00));
3543 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3544 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3545 || !(pCtx->fs.Attr.n.u1Granularity));
3546 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3547 || (pCtx->fs.Attr.n.u1Granularity));
3548 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3549 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3550 }
3551 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3552 {
3553 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3554 Assert(pCtx->gs.Attr.n.u1Present);
3555 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3556 Assert(!(pCtx->gs.Attr.u & 0xf00));
3557 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3558 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3559 || !(pCtx->gs.Attr.n.u1Granularity));
3560 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3561 || (pCtx->gs.Attr.n.u1Granularity));
3562 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3563 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3564 }
3565 /* 64-bit capable CPUs. */
3566# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3567 Assert(!(pCtx->cs.u64Base >> 32));
3568 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3569 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3570 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3571# endif
3572 }
3573 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3574 || ( CPUMIsGuestInRealModeEx(pCtx)
3575 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3576 {
3577 /* Real and v86 mode checks. */
3578 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3579 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3580 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3581 {
3582 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3583 }
3584 else
3585 {
3586 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3587 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3588 }
3589
3590 /* CS */
3591 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3592 Assert(pCtx->cs.u32Limit == 0xffff);
3593 Assert(u32CSAttr == 0xf3);
3594 /* SS */
3595 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3596 Assert(pCtx->ss.u32Limit == 0xffff);
3597 Assert(u32SSAttr == 0xf3);
3598 /* DS */
3599 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3600 Assert(pCtx->ds.u32Limit == 0xffff);
3601 Assert(u32DSAttr == 0xf3);
3602 /* ES */
3603 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3604 Assert(pCtx->es.u32Limit == 0xffff);
3605 Assert(u32ESAttr == 0xf3);
3606 /* FS */
3607 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3608 Assert(pCtx->fs.u32Limit == 0xffff);
3609 Assert(u32FSAttr == 0xf3);
3610 /* GS */
3611 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3612 Assert(pCtx->gs.u32Limit == 0xffff);
3613 Assert(u32GSAttr == 0xf3);
3614 /* 64-bit capable CPUs. */
3615# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3616 Assert(!(pCtx->cs.u64Base >> 32));
3617 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3618 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3619 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3620# endif
3621 }
3622}
3623#endif /* VBOX_STRICT */
3624
3625
3626/**
3627 * Writes a guest segment register into the guest-state area in the VMCS.
3628 *
3629 * @returns VBox status code.
3630 * @param pVCpu Pointer to the VMCPU.
3631 * @param idxSel Index of the selector in the VMCS.
3632 * @param idxLimit Index of the segment limit in the VMCS.
3633 * @param idxBase Index of the segment base in the VMCS.
3634 * @param idxAccess Index of the access rights of the segment in the VMCS.
3635 * @param pSelReg Pointer to the segment selector.
3636 * @param pCtx Pointer to the guest-CPU context.
3637 *
3638 * @remarks No-long-jump zone!!!
3639 */
3640static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3641 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3642{
3643 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3644 AssertRCReturn(rc, rc);
3645 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3646 AssertRCReturn(rc, rc);
3647 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3648 AssertRCReturn(rc, rc);
3649
3650 uint32_t u32Access = pSelReg->Attr.u;
3651 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3652 {
3653 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3654 u32Access = 0xf3;
3655 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3656 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3657 }
3658 else
3659 {
3660 /*
3661 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3662 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3663 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3664 * loaded in protected-mode have their attribute as 0.
3665 */
3666 if (!u32Access)
3667 u32Access = X86DESCATTR_UNUSABLE;
3668 }
3669
3670 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3671 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3672 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3673
3674 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3675 AssertRCReturn(rc, rc);
3676 return rc;
3677}
3678
3679
3680/**
3681 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3682 * into the guest-state area in the VMCS.
3683 *
3684 * @returns VBox status code.
3685 * @param pVM Pointer to the VM.
3686 * @param pVCPU Pointer to the VMCPU.
3687 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3688 * out-of-sync. Make sure to update the required fields
3689 * before using them.
3690 *
3691 * @remarks Requires CR0 (strict builds validation).
3692 * @remarks No-long-jump zone!!!
3693 */
3694static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3695{
3696 int rc = VERR_INTERNAL_ERROR_5;
3697 PVM pVM = pVCpu->CTX_SUFF(pVM);
3698
3699 /*
3700 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
3701 */
3702 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3703 {
3704 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3705 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3706 {
3707 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3708 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3709 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3710 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3711 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3712 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3713 }
3714
3715#ifdef VBOX_WITH_REM
3716 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3717 {
3718 Assert(pVM->hm.s.vmx.pRealModeTSS);
3719 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3720 if ( pVCpu->hm.s.vmx.fWasInRealMode
3721 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3722 {
3723 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3724 in real-mode (e.g. OpenBSD 4.0) */
3725 REMFlushTBs(pVM);
3726 Log4(("Load: Switch to protected mode detected!\n"));
3727 pVCpu->hm.s.vmx.fWasInRealMode = false;
3728 }
3729 }
3730#endif
3731 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3732 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3733 AssertRCReturn(rc, rc);
3734 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3735 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3736 AssertRCReturn(rc, rc);
3737 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3738 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3739 AssertRCReturn(rc, rc);
3740 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3741 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3742 AssertRCReturn(rc, rc);
3743 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3744 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3745 AssertRCReturn(rc, rc);
3746 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3747 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3748 AssertRCReturn(rc, rc);
3749
3750 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3751 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3752#ifdef VBOX_STRICT
3753 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3754#endif
3755 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3756 }
3757
3758 /*
3759 * Guest TR.
3760 */
3761 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3762 {
3763 /*
3764 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3765 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3766 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3767 */
3768 uint16_t u16Sel = 0;
3769 uint32_t u32Limit = 0;
3770 uint64_t u64Base = 0;
3771 uint32_t u32AccessRights = 0;
3772
3773 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3774 {
3775 u16Sel = pMixedCtx->tr.Sel;
3776 u32Limit = pMixedCtx->tr.u32Limit;
3777 u64Base = pMixedCtx->tr.u64Base;
3778 u32AccessRights = pMixedCtx->tr.Attr.u;
3779 }
3780 else
3781 {
3782 Assert(pVM->hm.s.vmx.pRealModeTSS);
3783 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3784
3785 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3786 RTGCPHYS GCPhys;
3787 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3788 AssertRCReturn(rc, rc);
3789
3790 X86DESCATTR DescAttr;
3791 DescAttr.u = 0;
3792 DescAttr.n.u1Present = 1;
3793 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3794
3795 u16Sel = 0;
3796 u32Limit = HM_VTX_TSS_SIZE;
3797 u64Base = GCPhys; /* in real-mode phys = virt. */
3798 u32AccessRights = DescAttr.u;
3799 }
3800
3801 /* Validate. */
3802 Assert(!(u16Sel & RT_BIT(2)));
3803 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3804 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3805 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3806 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3807 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3808 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3809 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3810 Assert( (u32Limit & 0xfff) == 0xfff
3811 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3812 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3813 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3814
3815 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3816 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3817 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3818 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3819
3820 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3821 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3822 }
3823
3824 /*
3825 * Guest GDTR.
3826 */
3827 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3828 {
3829 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3830 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3831
3832 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3833 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3834 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3835 }
3836
3837 /*
3838 * Guest LDTR.
3839 */
3840 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3841 {
3842 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3843 uint32_t u32Access = 0;
3844 if (!pMixedCtx->ldtr.Attr.u)
3845 u32Access = X86DESCATTR_UNUSABLE;
3846 else
3847 u32Access = pMixedCtx->ldtr.Attr.u;
3848
3849 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3850 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3851 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3852 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3853
3854 /* Validate. */
3855 if (!(u32Access & X86DESCATTR_UNUSABLE))
3856 {
3857 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3858 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3859 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3860 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3861 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3862 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3863 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3864 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3865 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3866 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3867 }
3868
3869 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3870 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3871 }
3872
3873 /*
3874 * Guest IDTR.
3875 */
3876 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3877 {
3878 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3879 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3880
3881 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3882 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3883 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3884 }
3885
3886 return VINF_SUCCESS;
3887}
3888
3889
3890/**
3891 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3892 * areas. These MSRs will automatically be loaded to the host CPU on every
3893 * successful VM entry and stored from the host CPU on every successful VM exit.
3894 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
3895 *
3896 * @returns VBox status code.
3897 * @param pVCpu Pointer to the VMCPU.
3898 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3899 * out-of-sync. Make sure to update the required fields
3900 * before using them.
3901 *
3902 * @remarks No-long-jump zone!!!
3903 */
3904static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3905{
3906 AssertPtr(pVCpu);
3907 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3908
3909 /*
3910 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3911 */
3912 int rc = VINF_SUCCESS;
3913 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3914 {
3915#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3916 PVM pVM = pVCpu->CTX_SUFF(pVM);
3917 PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3918 uint32_t cGuestMsrs = 0;
3919
3920 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3921 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3922 * when the guest really is in 64-bit mode. */
3923 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3924 if (fSupportsLongMode)
3925 {
3926 pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3927 pGuestMsr->u32Reserved = 0;
3928 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3929 pGuestMsr++; cGuestMsrs++;
3930 pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3931 pGuestMsr->u32Reserved = 0;
3932 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3933 pGuestMsr++; cGuestMsrs++;
3934 pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3935 pGuestMsr->u32Reserved = 0;
3936 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3937 pGuestMsr++; cGuestMsrs++;
3938 pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3939 pGuestMsr->u32Reserved = 0;
3940 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3941 pGuestMsr++; cGuestMsrs++;
3942 }
3943
3944 /*
3945 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3946 * load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3947 */
3948 if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3949 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3950 {
3951 pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3952 pGuestMsr->u32Reserved = 0;
3953 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3954 AssertRCReturn(rc, rc);
3955 pGuestMsr++; cGuestMsrs++;
3956 }
3957
3958 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3959 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3960 {
3961 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3962 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
3963 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3964 }
3965
3966 /* Update the VCPU's copy of the guest MSR count. */
3967 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3968 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3969 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3970#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3971
3972 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3973 }
3974
3975 /*
3976 * Guest Sysenter MSRs.
3977 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3978 * VM-exits on WRMSRs for these MSRs.
3979 */
3980 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3981 {
3982 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3983 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3984 }
3985 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3986 {
3987 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3988 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3989 }
3990 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3991 {
3992 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3993 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3994 }
3995
3996 return rc;
3997}
3998
3999
4000/**
4001 * Loads the guest activity state into the guest-state area in the VMCS.
4002 *
4003 * @returns VBox status code.
4004 * @param pVCpu Pointer to the VMCPU.
4005 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4006 * out-of-sync. Make sure to update the required fields
4007 * before using them.
4008 *
4009 * @remarks No-long-jump zone!!!
4010 */
4011static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4012{
4013 /** @todo See if we can make use of other states, e.g.
4014 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4015 int rc = VINF_SUCCESS;
4016 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
4017 {
4018 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4019 AssertRCReturn(rc, rc);
4020 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
4021 }
4022 return rc;
4023}
4024
4025
4026/**
4027 * Sets up the appropriate function to run guest code.
4028 *
4029 * @returns VBox status code.
4030 * @param pVCpu Pointer to the VMCPU.
4031 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4032 * out-of-sync. Make sure to update the required fields
4033 * before using them.
4034 *
4035 * @remarks No-long-jump zone!!!
4036 */
4037static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4038{
4039 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4040 {
4041#ifndef VBOX_ENABLE_64_BITS_GUESTS
4042 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4043#endif
4044 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4045#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4046 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4047 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4048#else
4049 /* 64-bit host or hybrid host. */
4050 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4051#endif
4052 }
4053 else
4054 {
4055 /* Guest is not in long mode, use the 32-bit handler. */
4056 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4057 }
4058 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4059 return VINF_SUCCESS;
4060}
4061
4062
4063/**
4064 * Wrapper for running the guest code in VT-x.
4065 *
4066 * @returns VBox strict status code.
4067 * @param pVM Pointer to the VM.
4068 * @param pVCpu Pointer to the VMCPU.
4069 * @param pCtx Pointer to the guest-CPU context.
4070 *
4071 * @remarks No-long-jump zone!!!
4072 */
4073DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4074{
4075 /*
4076 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4077 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4078 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4079 */
4080 const bool fResumeVM = !!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4081 /** @todo Add stats for resume vs launch. */
4082#ifdef VBOX_WITH_KERNEL_USING_XMM
4083 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4084#else
4085 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4086#endif
4087}
4088
4089
4090/**
4091 * Reports world-switch error and dumps some useful debug info.
4092 *
4093 * @param pVM Pointer to the VM.
4094 * @param pVCpu Pointer to the VMCPU.
4095 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4096 * @param pCtx Pointer to the guest-CPU context.
4097 * @param pVmxTransient Pointer to the VMX transient structure (only
4098 * exitReason updated).
4099 */
4100static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4101{
4102 Assert(pVM);
4103 Assert(pVCpu);
4104 Assert(pCtx);
4105 Assert(pVmxTransient);
4106 HMVMX_ASSERT_PREEMPT_SAFE();
4107
4108 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4109 switch (rcVMRun)
4110 {
4111 case VERR_VMX_INVALID_VMXON_PTR:
4112 AssertFailed();
4113 break;
4114 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4115 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4116 {
4117 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4118 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4119 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4120 AssertRC(rc);
4121
4122 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4123 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4124 Cannot do it here as we may have been long preempted. */
4125
4126#ifdef VBOX_STRICT
4127 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4128 pVmxTransient->uExitReason));
4129 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4130 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4131 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4132 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4133 else
4134 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4135
4136 /* VMX control bits. */
4137 uint32_t u32Val;
4138 uint64_t u64Val;
4139 HMVMXHCUINTREG uHCReg;
4140 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4141 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4142 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4143 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4144 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4145 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4146 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4147 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4148 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4149 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4150 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4151 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4152 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4153 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4154 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4155 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4156 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4157 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4158 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4159 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4160 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4161 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4162 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4163 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4164 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4165 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4166 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4167 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4168 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4169 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4170 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4171 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4172 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4173 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4174 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4175 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4176 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4177 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4178 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4179 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4180 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4181 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4182
4183 /* Guest bits. */
4184 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4185 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4186 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4187 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4188 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4189 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4190 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4191 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4192
4193 /* Host bits. */
4194 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4195 Log4(("Host CR0 %#RHr\n", uHCReg));
4196 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4197 Log4(("Host CR3 %#RHr\n", uHCReg));
4198 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4199 Log4(("Host CR4 %#RHr\n", uHCReg));
4200
4201 RTGDTR HostGdtr;
4202 PCX86DESCHC pDesc;
4203 ASMGetGDTR(&HostGdtr);
4204 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4205 Log4(("Host CS %#08x\n", u32Val));
4206 if (u32Val < HostGdtr.cbGdt)
4207 {
4208 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4209 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4210 }
4211
4212 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4213 Log4(("Host DS %#08x\n", u32Val));
4214 if (u32Val < HostGdtr.cbGdt)
4215 {
4216 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4217 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4218 }
4219
4220 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4221 Log4(("Host ES %#08x\n", u32Val));
4222 if (u32Val < HostGdtr.cbGdt)
4223 {
4224 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4225 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4226 }
4227
4228 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4229 Log4(("Host FS %#08x\n", u32Val));
4230 if (u32Val < HostGdtr.cbGdt)
4231 {
4232 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4233 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4234 }
4235
4236 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4237 Log4(("Host GS %#08x\n", u32Val));
4238 if (u32Val < HostGdtr.cbGdt)
4239 {
4240 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4241 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4242 }
4243
4244 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4245 Log4(("Host SS %#08x\n", u32Val));
4246 if (u32Val < HostGdtr.cbGdt)
4247 {
4248 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4249 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4250 }
4251
4252 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4253 Log4(("Host TR %#08x\n", u32Val));
4254 if (u32Val < HostGdtr.cbGdt)
4255 {
4256 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4257 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4258 }
4259
4260 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4261 Log4(("Host TR Base %#RHv\n", uHCReg));
4262 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4263 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4264 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4265 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4266 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4267 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4268 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4269 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4270 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4271 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4272 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4273 Log4(("Host RSP %#RHv\n", uHCReg));
4274 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4275 Log4(("Host RIP %#RHv\n", uHCReg));
4276# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4277 if (HMVMX_IS_64BIT_HOST_MODE())
4278 {
4279 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4280 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4281 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4282 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4283 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4284 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4285 }
4286# endif
4287#endif /* VBOX_STRICT */
4288 break;
4289 }
4290
4291 default:
4292 /* Impossible */
4293 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4294 break;
4295 }
4296 NOREF(pVM);
4297}
4298
4299
4300#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4301#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4302# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4303#endif
4304#ifdef VBOX_STRICT
4305static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4306{
4307 switch (idxField)
4308 {
4309 case VMX_VMCS_GUEST_RIP:
4310 case VMX_VMCS_GUEST_RSP:
4311 case VMX_VMCS_GUEST_SYSENTER_EIP:
4312 case VMX_VMCS_GUEST_SYSENTER_ESP:
4313 case VMX_VMCS_GUEST_GDTR_BASE:
4314 case VMX_VMCS_GUEST_IDTR_BASE:
4315 case VMX_VMCS_GUEST_CS_BASE:
4316 case VMX_VMCS_GUEST_DS_BASE:
4317 case VMX_VMCS_GUEST_ES_BASE:
4318 case VMX_VMCS_GUEST_FS_BASE:
4319 case VMX_VMCS_GUEST_GS_BASE:
4320 case VMX_VMCS_GUEST_SS_BASE:
4321 case VMX_VMCS_GUEST_LDTR_BASE:
4322 case VMX_VMCS_GUEST_TR_BASE:
4323 case VMX_VMCS_GUEST_CR3:
4324 return true;
4325 }
4326 return false;
4327}
4328
4329static bool hmR0VmxIsValidReadField(uint32_t idxField)
4330{
4331 switch (idxField)
4332 {
4333 /* Read-only fields. */
4334 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4335 return true;
4336 }
4337 /* Remaining readable fields should also be writable. */
4338 return hmR0VmxIsValidWriteField(idxField);
4339}
4340#endif /* VBOX_STRICT */
4341
4342
4343/**
4344 * Executes the specified handler in 64-bit mode.
4345 *
4346 * @returns VBox status code.
4347 * @param pVM Pointer to the VM.
4348 * @param pVCpu Pointer to the VMCPU.
4349 * @param pCtx Pointer to the guest CPU context.
4350 * @param enmOp The operation to perform.
4351 * @param cbParam Number of parameters.
4352 * @param paParam Array of 32-bit parameters.
4353 */
4354VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4355 uint32_t *paParam)
4356{
4357 int rc, rc2;
4358 PHMGLOBALCPUINFO pCpu;
4359 RTHCPHYS HCPhysCpuPage;
4360 RTCCUINTREG uOldEflags;
4361
4362 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4363 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4364 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4365 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4366
4367#ifdef VBOX_STRICT
4368 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4369 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4370
4371 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4372 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4373#endif
4374
4375 /* Disable interrupts. */
4376 uOldEflags = ASMIntDisableFlags();
4377
4378#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4379 RTCPUID idHostCpu = RTMpCpuId();
4380 CPUMR0SetLApic(pVCpu, idHostCpu);
4381#endif
4382
4383 pCpu = HMR0GetCurrentCpu();
4384 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4385
4386 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4387 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4388
4389 /* Leave VMX Root Mode. */
4390 VMXDisable();
4391
4392 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4393
4394 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4395 CPUMSetHyperEIP(pVCpu, enmOp);
4396 for (int i = (int)cbParam - 1; i >= 0; i--)
4397 CPUMPushHyper(pVCpu, paParam[i]);
4398
4399 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4400
4401 /* Call the switcher. */
4402 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4403 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4404
4405 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4406 /* Make sure the VMX instructions don't cause #UD faults. */
4407 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4408
4409 /* Re-enter VMX Root Mode */
4410 rc2 = VMXEnable(HCPhysCpuPage);
4411 if (RT_FAILURE(rc2))
4412 {
4413 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4414 ASMSetFlags(uOldEflags);
4415 return rc2;
4416 }
4417
4418 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4419 AssertRC(rc2);
4420 Assert(!(ASMGetFlags() & X86_EFL_IF));
4421 ASMSetFlags(uOldEflags);
4422 return rc;
4423}
4424
4425
4426/**
4427 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4428 * supporting 64-bit guests.
4429 *
4430 * @returns VBox status code.
4431 * @param fResume Whether to VMLAUNCH or VMRESUME.
4432 * @param pCtx Pointer to the guest-CPU context.
4433 * @param pCache Pointer to the VMCS cache.
4434 * @param pVM Pointer to the VM.
4435 * @param pVCpu Pointer to the VMCPU.
4436 */
4437DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4438{
4439 uint32_t aParam[6];
4440 PHMGLOBALCPUINFO pCpu = NULL;
4441 RTHCPHYS HCPhysCpuPage = 0;
4442 int rc = VERR_INTERNAL_ERROR_5;
4443
4444 pCpu = HMR0GetCurrentCpu();
4445 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4446
4447#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4448 pCache->uPos = 1;
4449 pCache->interPD = PGMGetInterPaeCR3(pVM);
4450 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4451#endif
4452
4453#ifdef VBOX_STRICT
4454 pCache->TestIn.HCPhysCpuPage = 0;
4455 pCache->TestIn.HCPhysVmcs = 0;
4456 pCache->TestIn.pCache = 0;
4457 pCache->TestOut.HCPhysVmcs = 0;
4458 pCache->TestOut.pCache = 0;
4459 pCache->TestOut.pCtx = 0;
4460 pCache->TestOut.eflags = 0;
4461#endif
4462
4463 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4464 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4465 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4466 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4467 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4468 aParam[5] = 0;
4469
4470#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4471 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4472 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4473#endif
4474 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4475
4476#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4477 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4478 Assert(pCtx->dr[4] == 10);
4479 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4480#endif
4481
4482#ifdef VBOX_STRICT
4483 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4484 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4485 pVCpu->hm.s.vmx.HCPhysVmcs));
4486 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4487 pCache->TestOut.HCPhysVmcs));
4488 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4489 pCache->TestOut.pCache));
4490 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4491 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4492 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4493 pCache->TestOut.pCtx));
4494 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4495#endif
4496 return rc;
4497}
4498
4499
4500/**
4501 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4502 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4503 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4504 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4505 *
4506 * @returns VBox status code.
4507 * @param pVM Pointer to the VM.
4508 * @param pVCpu Pointer to the VMCPU.
4509 */
4510static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4511{
4512#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4513{ \
4514 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4515 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4516 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4517 ++cReadFields; \
4518}
4519
4520 AssertPtr(pVM);
4521 AssertPtr(pVCpu);
4522 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4523 uint32_t cReadFields = 0;
4524
4525 /*
4526 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4527 * and serve to indicate exceptions to the rules.
4528 */
4529
4530 /* Guest-natural selector base fields. */
4531#if 0
4532 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4533 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4534 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4535#endif
4536 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4537 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4538 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4539 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4540 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4541 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4542 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4543 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4544 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4545 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4546 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4547 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4548#if 0
4549 /* Unused natural width guest-state fields. */
4550 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4551 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4552#endif
4553 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4554 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4555
4556 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4557#if 0
4558 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4559 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4560 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4561 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4562 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4563 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4564 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4565 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4566 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4567#endif
4568
4569 /* Natural width guest-state fields. */
4570 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4571#if 0
4572 /* Currently unused field. */
4573 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4574#endif
4575
4576 if (pVM->hm.s.fNestedPaging)
4577 {
4578 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4579 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4580 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4581 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4582 }
4583 else
4584 {
4585 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4586 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4587 }
4588
4589#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4590 return VINF_SUCCESS;
4591}
4592
4593
4594/**
4595 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4596 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4597 * darwin, running 64-bit guests).
4598 *
4599 * @returns VBox status code.
4600 * @param pVCpu Pointer to the VMCPU.
4601 * @param idxField The VMCS field encoding.
4602 * @param u64Val 16, 32 or 64 bits value.
4603 */
4604VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4605{
4606 int rc;
4607 switch (idxField)
4608 {
4609 /*
4610 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4611 */
4612 /* 64-bit Control fields. */
4613 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4614 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4615 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4616 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4617 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4618 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4619 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4620 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4621 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4622 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4623 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4624 case VMX_VMCS64_CTRL_EPTP_FULL:
4625 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4626 /* 64-bit Guest-state fields. */
4627 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4628 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4629 case VMX_VMCS64_GUEST_PAT_FULL:
4630 case VMX_VMCS64_GUEST_EFER_FULL:
4631 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4632 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4633 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4634 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4635 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4636 /* 64-bit Host-state fields. */
4637 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4638 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4639 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4640 {
4641 rc = VMXWriteVmcs32(idxField, u64Val);
4642 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4643 break;
4644 }
4645
4646 /*
4647 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4648 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4649 */
4650 /* Natural-width Guest-state fields. */
4651 case VMX_VMCS_GUEST_CR3:
4652 case VMX_VMCS_GUEST_ES_BASE:
4653 case VMX_VMCS_GUEST_CS_BASE:
4654 case VMX_VMCS_GUEST_SS_BASE:
4655 case VMX_VMCS_GUEST_DS_BASE:
4656 case VMX_VMCS_GUEST_FS_BASE:
4657 case VMX_VMCS_GUEST_GS_BASE:
4658 case VMX_VMCS_GUEST_LDTR_BASE:
4659 case VMX_VMCS_GUEST_TR_BASE:
4660 case VMX_VMCS_GUEST_GDTR_BASE:
4661 case VMX_VMCS_GUEST_IDTR_BASE:
4662 case VMX_VMCS_GUEST_RSP:
4663 case VMX_VMCS_GUEST_RIP:
4664 case VMX_VMCS_GUEST_SYSENTER_ESP:
4665 case VMX_VMCS_GUEST_SYSENTER_EIP:
4666 {
4667 if (!(u64Val >> 32))
4668 {
4669 /* If this field is 64-bit, VT-x will zero out the top bits. */
4670 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4671 }
4672 else
4673 {
4674 /* Assert that only the 32->64 switcher case should ever come here. */
4675 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4676 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4677 }
4678 break;
4679 }
4680
4681 default:
4682 {
4683 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4684 rc = VERR_INVALID_PARAMETER;
4685 break;
4686 }
4687 }
4688 AssertRCReturn(rc, rc);
4689 return rc;
4690}
4691
4692
4693/**
4694 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4695 * hosts (except darwin) for 64-bit guests.
4696 *
4697 * @param pVCpu Pointer to the VMCPU.
4698 * @param idxField The VMCS field encoding.
4699 * @param u64Val 16, 32 or 64 bits value.
4700 */
4701VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4702{
4703 AssertPtr(pVCpu);
4704 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4705
4706 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4707 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4708
4709 /* Make sure there are no duplicates. */
4710 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4711 {
4712 if (pCache->Write.aField[i] == idxField)
4713 {
4714 pCache->Write.aFieldVal[i] = u64Val;
4715 return VINF_SUCCESS;
4716 }
4717 }
4718
4719 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4720 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4721 pCache->Write.cValidEntries++;
4722 return VINF_SUCCESS;
4723}
4724
4725/* Enable later when the assembly code uses these as callbacks. */
4726#if 0
4727/*
4728 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4729 *
4730 * @param pVCpu Pointer to the VMCPU.
4731 * @param pCache Pointer to the VMCS cache.
4732 *
4733 * @remarks No-long-jump zone!!!
4734 */
4735VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4736{
4737 AssertPtr(pCache);
4738 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4739 {
4740 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4741 AssertRC(rc);
4742 }
4743 pCache->Write.cValidEntries = 0;
4744}
4745
4746
4747/**
4748 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
4749 *
4750 * @param pVCpu Pointer to the VMCPU.
4751 * @param pCache Pointer to the VMCS cache.
4752 *
4753 * @remarks No-long-jump zone!!!
4754 */
4755VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4756{
4757 AssertPtr(pCache);
4758 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4759 {
4760 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4761 AssertRC(rc);
4762 }
4763}
4764#endif
4765#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4766
4767
4768/**
4769 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4770 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4771 * timer.
4772 *
4773 * @returns VBox status code.
4774 * @param pVCpu Pointer to the VMCPU.
4775 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4776 * out-of-sync. Make sure to update the required fields
4777 * before using them.
4778 * @remarks No-long-jump zone!!!
4779 */
4780static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4781{
4782 int rc = VERR_INTERNAL_ERROR_5;
4783 bool fOffsettedTsc = false;
4784 PVM pVM = pVCpu->CTX_SUFF(pVM);
4785 if (pVM->hm.s.vmx.fUsePreemptTimer)
4786 {
4787 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4788
4789 /* Make sure the returned values have sane upper and lower boundaries. */
4790 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4791 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4792 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4793 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4794
4795 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4796 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4797 }
4798 else
4799 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4800
4801 if (fOffsettedTsc)
4802 {
4803 uint64_t u64CurTSC = ASMReadTSC();
4804 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4805 {
4806 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4807 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4808
4809 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4810 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4811 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4812 }
4813 else
4814 {
4815 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4816 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4817 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4818 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4819 }
4820 }
4821 else
4822 {
4823 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4824 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4825 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4826 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4827 }
4828}
4829
4830
4831/**
4832 * Determines if an exception is a contributory exception. Contributory
4833 * exceptions are ones which can cause double-faults. Page-fault is
4834 * intentionally not included here as it's a conditional contributory exception.
4835 *
4836 * @returns true if the exception is contributory, false otherwise.
4837 * @param uVector The exception vector.
4838 */
4839DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4840{
4841 switch (uVector)
4842 {
4843 case X86_XCPT_GP:
4844 case X86_XCPT_SS:
4845 case X86_XCPT_NP:
4846 case X86_XCPT_TS:
4847 case X86_XCPT_DE:
4848 return true;
4849 default:
4850 break;
4851 }
4852 return false;
4853}
4854
4855
4856/**
4857 * Sets an event as a pending event to be injected into the guest.
4858 *
4859 * @param pVCpu Pointer to the VMCPU.
4860 * @param u32IntrInfo The VM-entry interruption-information field.
4861 * @param cbInstr The VM-entry instruction length in bytes (for software
4862 * interrupts, exceptions and privileged software
4863 * exceptions).
4864 * @param u32ErrCode The VM-entry exception error code.
4865 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4866 * page-fault.
4867 *
4868 * @remarks Statistics counter assumes this is a guest event being injected or
4869 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4870 * always incremented.
4871 */
4872DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4873 RTGCUINTPTR GCPtrFaultAddress)
4874{
4875 Assert(!pVCpu->hm.s.Event.fPending);
4876 pVCpu->hm.s.Event.fPending = true;
4877 pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4878 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4879 pVCpu->hm.s.Event.cbInstr = cbInstr;
4880 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4881
4882 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4883}
4884
4885
4886/**
4887 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4888 *
4889 * @param pVCpu Pointer to the VMCPU.
4890 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4891 * out-of-sync. Make sure to update the required fields
4892 * before using them.
4893 */
4894DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4895{
4896 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
4897 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4898 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4899 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
4900}
4901
4902
4903/**
4904 * Handle a condition that occurred while delivering an event through the guest
4905 * IDT.
4906 *
4907 * @returns VBox status code (informational error codes included).
4908 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
4909 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4910 * continue execution of the guest which will delivery the #DF.
4911 * @retval VINF_EM_RESET if we detected a triple-fault condition.
4912 *
4913 * @param pVCpu Pointer to the VMCPU.
4914 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4915 * out-of-sync. Make sure to update the required fields
4916 * before using them.
4917 * @param pVmxTransient Pointer to the VMX transient structure.
4918 *
4919 * @remarks No-long-jump zone!!!
4920 */
4921static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4922{
4923 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4924 AssertRC(rc);
4925 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4926 {
4927 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4928 AssertRCReturn(rc, rc);
4929
4930 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4931 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4932 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4933
4934 typedef enum
4935 {
4936 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4937 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4938 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4939 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4940 } VMXREFLECTXCPT;
4941
4942 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4943 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4944 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4945 {
4946 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4947 {
4948 enmReflect = VMXREFLECTXCPT_XCPT;
4949#ifdef VBOX_STRICT
4950 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4951 && uExitVector == X86_XCPT_PF)
4952 {
4953 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4954 }
4955#endif
4956 if ( uExitVector == X86_XCPT_PF
4957 && uIdtVector == X86_XCPT_PF)
4958 {
4959 pVmxTransient->fVectoringPF = true;
4960 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4961 }
4962 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4963 && hmR0VmxIsContributoryXcpt(uExitVector)
4964 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
4965 || uIdtVector == X86_XCPT_PF))
4966 {
4967 enmReflect = VMXREFLECTXCPT_DF;
4968 }
4969 else if (uIdtVector == X86_XCPT_DF)
4970 enmReflect = VMXREFLECTXCPT_TF;
4971 }
4972 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4973 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4974 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4975 {
4976 /*
4977 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4978 * (whatever they are) as they reoccur when restarting the instruction.
4979 */
4980 enmReflect = VMXREFLECTXCPT_XCPT;
4981 }
4982 }
4983 else
4984 {
4985 /*
4986 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
4987 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
4988 * original exception to the guest after handling the VM-exit.
4989 */
4990 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
4991 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
4992 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
4993 {
4994 enmReflect = VMXREFLECTXCPT_XCPT;
4995 }
4996 }
4997
4998 switch (enmReflect)
4999 {
5000 case VMXREFLECTXCPT_XCPT:
5001 {
5002 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5003 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5004 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5005
5006 uint32_t u32ErrCode = 0;
5007 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5008 {
5009 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5010 AssertRCReturn(rc, rc);
5011 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5012 }
5013
5014 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5015 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5016 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5017 rc = VINF_SUCCESS;
5018 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5019 pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
5020
5021 break;
5022 }
5023
5024 case VMXREFLECTXCPT_DF:
5025 {
5026 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5027 rc = VINF_HM_DOUBLE_FAULT;
5028 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5029 pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
5030
5031 break;
5032 }
5033
5034 case VMXREFLECTXCPT_TF:
5035 {
5036 rc = VINF_EM_RESET;
5037 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5038 uExitVector));
5039 break;
5040 }
5041
5042 default:
5043 Assert(rc == VINF_SUCCESS);
5044 break;
5045 }
5046 }
5047 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5048 return rc;
5049}
5050
5051
5052/**
5053 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5054 *
5055 * @returns VBox status code.
5056 * @param pVCpu Pointer to the VMCPU.
5057 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5058 * out-of-sync. Make sure to update the required fields
5059 * before using them.
5060 *
5061 * @remarks No-long-jump zone!!!
5062 */
5063static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5064{
5065 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5066 {
5067 uint32_t uVal = 0;
5068 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5069 AssertRCReturn(rc, rc);
5070 uint32_t uShadow = 0;
5071 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5072 AssertRCReturn(rc, rc);
5073
5074 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5075 CPUMSetGuestCR0(pVCpu, uVal);
5076 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5077 }
5078 return VINF_SUCCESS;
5079}
5080
5081
5082/**
5083 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5084 *
5085 * @returns VBox status code.
5086 * @param pVCpu Pointer to the VMCPU.
5087 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5088 * out-of-sync. Make sure to update the required fields
5089 * before using them.
5090 *
5091 * @remarks No-long-jump zone!!!
5092 */
5093static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5094{
5095 int rc = VINF_SUCCESS;
5096 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5097 {
5098 uint32_t uVal = 0;
5099 uint32_t uShadow = 0;
5100 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5101 AssertRCReturn(rc, rc);
5102 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5103 AssertRCReturn(rc, rc);
5104
5105 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5106 CPUMSetGuestCR4(pVCpu, uVal);
5107 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5108 }
5109 return rc;
5110}
5111
5112
5113/**
5114 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5115 *
5116 * @returns VBox status code.
5117 * @param pVCpu Pointer to the VMCPU.
5118 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5119 * out-of-sync. Make sure to update the required fields
5120 * before using them.
5121 *
5122 * @remarks No-long-jump zone!!!
5123 */
5124static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5125{
5126 int rc = VINF_SUCCESS;
5127 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5128 {
5129 uint64_t u64Val = 0;
5130 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5131 AssertRCReturn(rc, rc);
5132
5133 pMixedCtx->rip = u64Val;
5134 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5135 }
5136 return rc;
5137}
5138
5139
5140/**
5141 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5142 *
5143 * @returns VBox status code.
5144 * @param pVCpu Pointer to the VMCPU.
5145 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5146 * out-of-sync. Make sure to update the required fields
5147 * before using them.
5148 *
5149 * @remarks No-long-jump zone!!!
5150 */
5151static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5152{
5153 int rc = VINF_SUCCESS;
5154 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5155 {
5156 uint64_t u64Val = 0;
5157 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5158 AssertRCReturn(rc, rc);
5159
5160 pMixedCtx->rsp = u64Val;
5161 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5162 }
5163 return rc;
5164}
5165
5166
5167/**
5168 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5169 *
5170 * @returns VBox status code.
5171 * @param pVCpu Pointer to the VMCPU.
5172 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5173 * out-of-sync. Make sure to update the required fields
5174 * before using them.
5175 *
5176 * @remarks No-long-jump zone!!!
5177 */
5178static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5179{
5180 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5181 {
5182 uint32_t uVal = 0;
5183 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5184 AssertRCReturn(rc, rc);
5185
5186 pMixedCtx->eflags.u32 = uVal;
5187 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5188 {
5189 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5190 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5191
5192 pMixedCtx->eflags.Bits.u1VM = 0;
5193 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5194 }
5195
5196 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5197 }
5198 return VINF_SUCCESS;
5199}
5200
5201
5202/**
5203 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5204 * guest-CPU context.
5205 */
5206DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5207{
5208 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5209 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5210 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5211 return rc;
5212}
5213
5214
5215/**
5216 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5217 * from the guest-state area in the VMCS.
5218 *
5219 * @param pVCpu Pointer to the VMCPU.
5220 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5221 * out-of-sync. Make sure to update the required fields
5222 * before using them.
5223 *
5224 * @remarks No-long-jump zone!!!
5225 */
5226static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5227{
5228 uint32_t uIntrState = 0;
5229 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5230 AssertRC(rc);
5231
5232 if (!uIntrState)
5233 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5234 else
5235 {
5236 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5237 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5238 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5239 AssertRC(rc);
5240 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5241 AssertRC(rc);
5242
5243 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5244 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5245 }
5246}
5247
5248
5249/**
5250 * Saves the guest's activity state.
5251 *
5252 * @returns VBox status code.
5253 * @param pVCpu Pointer to the VMCPU.
5254 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5255 * out-of-sync. Make sure to update the required fields
5256 * before using them.
5257 *
5258 * @remarks No-long-jump zone!!!
5259 */
5260static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5261{
5262 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5263 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5264 return VINF_SUCCESS;
5265}
5266
5267
5268/**
5269 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5270 * the current VMCS into the guest-CPU context.
5271 *
5272 * @returns VBox status code.
5273 * @param pVCpu Pointer to the VMCPU.
5274 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5275 * out-of-sync. Make sure to update the required fields
5276 * before using them.
5277 *
5278 * @remarks No-long-jump zone!!!
5279 */
5280static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5281{
5282 int rc = VINF_SUCCESS;
5283 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5284 {
5285 uint32_t u32Val = 0;
5286 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5287 pMixedCtx->SysEnter.cs = u32Val;
5288 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5289 }
5290
5291 uint64_t u64Val = 0;
5292 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5293 {
5294 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5295 pMixedCtx->SysEnter.eip = u64Val;
5296 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5297 }
5298 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5299 {
5300 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5301 pMixedCtx->SysEnter.esp = u64Val;
5302 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5303 }
5304 return rc;
5305}
5306
5307
5308/**
5309 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5310 * context.
5311 *
5312 * @returns VBox status code.
5313 * @param pVCpu Pointer to the VMCPU.
5314 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5315 * out-of-sync. Make sure to update the required fields
5316 * before using them.
5317 *
5318 * @remarks No-long-jump zone!!!
5319 */
5320static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5321{
5322 int rc = VINF_SUCCESS;
5323 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5324 {
5325 uint64_t u64Val = 0;
5326 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5327 pMixedCtx->fs.u64Base = u64Val;
5328 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5329 }
5330 return rc;
5331}
5332
5333
5334/**
5335 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5336 * context.
5337 *
5338 * @returns VBox status code.
5339 * @param pVCpu Pointer to the VMCPU.
5340 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5341 * out-of-sync. Make sure to update the required fields
5342 * before using them.
5343 *
5344 * @remarks No-long-jump zone!!!
5345 */
5346static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5347{
5348 int rc = VINF_SUCCESS;
5349 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5350 {
5351 uint64_t u64Val = 0;
5352 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5353 pMixedCtx->gs.u64Base = u64Val;
5354 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5355 }
5356 return rc;
5357}
5358
5359
5360/**
5361 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5362 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5363 * and TSC_AUX.
5364 *
5365 * @returns VBox status code.
5366 * @param pVCpu Pointer to the VMCPU.
5367 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5368 * out-of-sync. Make sure to update the required fields
5369 * before using them.
5370 *
5371 * @remarks No-long-jump zone!!!
5372 */
5373static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5374{
5375 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5376 return VINF_SUCCESS;
5377
5378#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5379 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5380 {
5381 PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5382 pMsr += i;
5383 switch (pMsr->u32IndexMSR)
5384 {
5385 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5386 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5387 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5388 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5389 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5390 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5391 default:
5392 {
5393 AssertFailed();
5394 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5395 }
5396 }
5397 }
5398#endif
5399
5400 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5401 return VINF_SUCCESS;
5402}
5403
5404
5405/**
5406 * Saves the guest control registers from the current VMCS into the guest-CPU
5407 * context.
5408 *
5409 * @returns VBox status code.
5410 * @param pVCpu Pointer to the VMCPU.
5411 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5412 * out-of-sync. Make sure to update the required fields
5413 * before using them.
5414 *
5415 * @remarks No-long-jump zone!!!
5416 */
5417static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5418{
5419 /* Guest CR0. Guest FPU. */
5420 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5421 AssertRCReturn(rc, rc);
5422
5423 /* Guest CR4. */
5424 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5425 AssertRCReturn(rc, rc);
5426
5427 /* Guest CR2 - updated always during the world-switch or in #PF. */
5428 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5429 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5430 {
5431 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5432 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5433
5434 PVM pVM = pVCpu->CTX_SUFF(pVM);
5435 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5436 || ( pVM->hm.s.fNestedPaging
5437 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5438 {
5439 uint64_t u64Val = 0;
5440 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5441 if (pMixedCtx->cr3 != u64Val)
5442 {
5443 CPUMSetGuestCR3(pVCpu, u64Val);
5444 if (VMMRZCallRing3IsEnabled(pVCpu))
5445 {
5446 PGMUpdateCR3(pVCpu, u64Val);
5447 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5448 }
5449 else
5450 {
5451 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5452 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5453 }
5454 }
5455
5456 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5457 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5458 {
5459 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5460 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5461 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5462 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5463
5464 if (VMMRZCallRing3IsEnabled(pVCpu))
5465 {
5466 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5467 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5468 }
5469 else
5470 {
5471 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5472 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5473 }
5474 }
5475 }
5476
5477 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5478 }
5479
5480 /*
5481 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5482 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5483 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5484 *
5485 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5486 */
5487 if (VMMRZCallRing3IsEnabled(pVCpu))
5488 {
5489 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5490 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5491
5492 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5493 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5494
5495 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5496 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5497 }
5498
5499 return rc;
5500}
5501
5502
5503/**
5504 * Reads a guest segment register from the current VMCS into the guest-CPU
5505 * context.
5506 *
5507 * @returns VBox status code.
5508 * @param pVCpu Pointer to the VMCPU.
5509 * @param idxSel Index of the selector in the VMCS.
5510 * @param idxLimit Index of the segment limit in the VMCS.
5511 * @param idxBase Index of the segment base in the VMCS.
5512 * @param idxAccess Index of the access rights of the segment in the VMCS.
5513 * @param pSelReg Pointer to the segment selector.
5514 *
5515 * @remarks No-long-jump zone!!!
5516 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5517 * macro as that takes care of whether to read from the VMCS cache or
5518 * not.
5519 */
5520DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5521 PCPUMSELREG pSelReg)
5522{
5523 uint32_t u32Val = 0;
5524 int rc = VMXReadVmcs32(idxSel, &u32Val);
5525 AssertRCReturn(rc, rc);
5526 pSelReg->Sel = (uint16_t)u32Val;
5527 pSelReg->ValidSel = (uint16_t)u32Val;
5528 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5529
5530 rc = VMXReadVmcs32(idxLimit, &u32Val);
5531 AssertRCReturn(rc, rc);
5532 pSelReg->u32Limit = u32Val;
5533
5534 uint64_t u64Val = 0;
5535 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5536 AssertRCReturn(rc, rc);
5537 pSelReg->u64Base = u64Val;
5538
5539 rc = VMXReadVmcs32(idxAccess, &u32Val);
5540 AssertRCReturn(rc, rc);
5541 pSelReg->Attr.u = u32Val;
5542
5543 /*
5544 * If VT-x marks the segment as unusable, most other bits remain undefined:
5545 * - For CS the L, D and G bits have meaning.
5546 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5547 * - For the remaining data segments no bits are defined.
5548 *
5549 * The present bit and the unusable bit has been observed to be set at the
5550 * same time (the selector was supposed to invalid as we started executing
5551 * a V8086 interrupt in ring-0).
5552 *
5553 * What should be important for the rest of the VBox code that the P bit is
5554 * cleared. Some of the other VBox code recognizes the unusable bit, but
5555 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5556 * safe side here, we'll strip off P and other bits we don't care about. If
5557 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5558 *
5559 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5560 */
5561 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5562 {
5563 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5564
5565 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5566 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5567 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5568
5569 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5570#ifdef DEBUG_bird
5571 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5572 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5573 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5574#endif
5575 }
5576 return VINF_SUCCESS;
5577}
5578
5579
5580#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5581# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5582 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5583 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5584#else
5585# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5586 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5587 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5588#endif
5589
5590
5591/**
5592 * Saves the guest segment registers from the current VMCS into the guest-CPU
5593 * context.
5594 *
5595 * @returns VBox status code.
5596 * @param pVCpu Pointer to the VMCPU.
5597 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5598 * out-of-sync. Make sure to update the required fields
5599 * before using them.
5600 *
5601 * @remarks No-long-jump zone!!!
5602 */
5603static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5604{
5605 /* Guest segment registers. */
5606 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5607 {
5608 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5609 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5610 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5611 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5612 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5613 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5614 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5615
5616 /* Restore segment attributes for real-on-v86 mode hack. */
5617 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5618 {
5619 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5620 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5621 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5622 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5623 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5624 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5625 }
5626 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5627 }
5628
5629 return VINF_SUCCESS;
5630}
5631
5632
5633/**
5634 * Saves the guest descriptor table registers and task register from the current
5635 * VMCS into the guest-CPU context.
5636 *
5637 * @returns VBox status code.
5638 * @param pVCpu Pointer to the VMCPU.
5639 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5640 * out-of-sync. Make sure to update the required fields
5641 * before using them.
5642 *
5643 * @remarks No-long-jump zone!!!
5644 */
5645static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5646{
5647 int rc = VINF_SUCCESS;
5648
5649 /* Guest LDTR. */
5650 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5651 {
5652 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5653 AssertRCReturn(rc, rc);
5654 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5655 }
5656
5657 /* Guest GDTR. */
5658 uint64_t u64Val = 0;
5659 uint32_t u32Val = 0;
5660 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5661 {
5662 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5663 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5664 pMixedCtx->gdtr.pGdt = u64Val;
5665 pMixedCtx->gdtr.cbGdt = u32Val;
5666 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
5667 }
5668
5669 /* Guest IDTR. */
5670 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5671 {
5672 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5673 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5674 pMixedCtx->idtr.pIdt = u64Val;
5675 pMixedCtx->idtr.cbIdt = u32Val;
5676 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
5677 }
5678
5679 /* Guest TR. */
5680 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5681 {
5682 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5683 AssertRCReturn(rc, rc);
5684
5685 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5686 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5687 {
5688 rc = VMXLOCAL_READ_SEG(TR, tr);
5689 AssertRCReturn(rc, rc);
5690 }
5691 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
5692 }
5693 return rc;
5694}
5695
5696#undef VMXLOCAL_READ_SEG
5697
5698
5699/**
5700 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5701 * context.
5702 *
5703 * @returns VBox status code.
5704 * @param pVCpu Pointer to the VMCPU.
5705 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5706 * out-of-sync. Make sure to update the required fields
5707 * before using them.
5708 *
5709 * @remarks No-long-jump zone!!!
5710 */
5711static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5712{
5713 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5714 {
5715 if (!CPUMIsHyperDebugStateActive(pVCpu))
5716 {
5717 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5718 uint32_t u32Val;
5719 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5720 pMixedCtx->dr[7] = u32Val;
5721 }
5722
5723 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
5724 }
5725 return VINF_SUCCESS;
5726}
5727
5728
5729/**
5730 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
5731 *
5732 * @returns VBox status code.
5733 * @param pVCpu Pointer to the VMCPU.
5734 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5735 * out-of-sync. Make sure to update the required fields
5736 * before using them.
5737 *
5738 * @remarks No-long-jump zone!!!
5739 */
5740static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5741{
5742 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5743 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
5744 return VINF_SUCCESS;
5745}
5746
5747
5748/**
5749 * Saves the entire guest state from the currently active VMCS into the
5750 * guest-CPU context. This essentially VMREADs all guest-data.
5751 *
5752 * @returns VBox status code.
5753 * @param pVCpu Pointer to the VMCPU.
5754 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5755 * out-of-sync. Make sure to update the required fields
5756 * before using them.
5757 */
5758static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5759{
5760 Assert(pVCpu);
5761 Assert(pMixedCtx);
5762
5763 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5764 return VINF_SUCCESS;
5765
5766 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5767 there is no real need to. */
5768 if (VMMRZCallRing3IsEnabled(pVCpu))
5769 VMMR0LogFlushDisable(pVCpu);
5770 else
5771 Assert(VMMR0IsLogFlushDisabled(pVCpu));
5772 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5773
5774 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5775 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5776
5777 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5778 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5779
5780 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5781 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5782
5783 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5784 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5785
5786 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5787 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5788
5789 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5790 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5791
5792 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5793 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5794
5795 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5796 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5797
5798 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5799 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5800
5801 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5802 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5803
5804 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5805 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5806
5807 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5808 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5809
5810 if (VMMRZCallRing3IsEnabled(pVCpu))
5811 VMMR0LogFlushEnable(pVCpu);
5812
5813 return rc;
5814}
5815
5816
5817/**
5818 * Check per-VM and per-VCPU force flag actions that require us to go back to
5819 * ring-3 for one reason or another.
5820 *
5821 * @returns VBox status code (information status code included).
5822 * @retval VINF_SUCCESS if we don't have any actions that require going back to
5823 * ring-3.
5824 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5825 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5826 * interrupts)
5827 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5828 * all EMTs to be in ring-3.
5829 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5830 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5831 * to the EM loop.
5832 *
5833 * @param pVM Pointer to the VM.
5834 * @param pVCpu Pointer to the VMCPU.
5835 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5836 * out-of-sync. Make sure to update the required fields
5837 * before using them.
5838 */
5839static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5840{
5841 Assert(VMMRZCallRing3IsEnabled(pVCpu));
5842
5843 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5844 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5845 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5846 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5847 {
5848 /* We need the control registers now, make sure the guest-CPU context is updated. */
5849 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5850 AssertRCReturn(rc3, rc3);
5851
5852 /* Pending HM CR3 sync. */
5853 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5854 {
5855 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5856 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
5857 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5858 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5859 }
5860
5861 /* Pending HM PAE PDPEs. */
5862 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5863 {
5864 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5865 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5866 }
5867
5868 /* Pending PGM C3 sync. */
5869 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5870 {
5871 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5872 if (rc2 != VINF_SUCCESS)
5873 {
5874 AssertRC(rc2);
5875 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5876 return rc2;
5877 }
5878 }
5879
5880 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5881 /* -XXX- what was that about single stepping? */
5882 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5883 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5884 {
5885 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5886 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5887 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5888 return rc2;
5889 }
5890
5891 /* Pending VM request packets, such as hardware interrupts. */
5892 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5893 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5894 {
5895 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5896 return VINF_EM_PENDING_REQUEST;
5897 }
5898
5899 /* Pending PGM pool flushes. */
5900 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5901 {
5902 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5903 return VINF_PGM_POOL_FLUSH_PENDING;
5904 }
5905
5906 /* Pending DMA requests. */
5907 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5908 {
5909 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5910 return VINF_EM_RAW_TO_R3;
5911 }
5912 }
5913
5914 /* Paranoia. */
5915 return VINF_SUCCESS;
5916}
5917
5918
5919/**
5920 * Converts any TRPM trap into a pending HM event. This is typically used when
5921 * entering from ring-3 (not longjmp returns).
5922 *
5923 * @param pVCpu Pointer to the VMCPU.
5924 */
5925static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5926{
5927 Assert(TRPMHasTrap(pVCpu));
5928 Assert(!pVCpu->hm.s.Event.fPending);
5929
5930 uint8_t uVector;
5931 TRPMEVENT enmTrpmEvent;
5932 RTGCUINT uErrCode;
5933 RTGCUINTPTR GCPtrFaultAddress;
5934 uint8_t cbInstr;
5935
5936 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5937 AssertRC(rc);
5938
5939 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5940 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
5941 if (enmTrpmEvent == TRPM_TRAP)
5942 {
5943 switch (uVector)
5944 {
5945 case X86_XCPT_BP:
5946 case X86_XCPT_OF:
5947 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5948 break;
5949
5950 case X86_XCPT_PF:
5951 case X86_XCPT_DF:
5952 case X86_XCPT_TS:
5953 case X86_XCPT_NP:
5954 case X86_XCPT_SS:
5955 case X86_XCPT_GP:
5956 case X86_XCPT_AC:
5957 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5958 /* no break! */
5959 default:
5960 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5961 break;
5962 }
5963 }
5964 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5965 {
5966 if (uVector == X86_XCPT_NMI)
5967 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5968 else
5969 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5970 }
5971 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5972 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5973 else
5974 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5975
5976 rc = TRPMResetTrap(pVCpu);
5977 AssertRC(rc);
5978 Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5979 u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5980
5981 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5982 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
5983}
5984
5985
5986/**
5987 * Converts any pending HM event into a TRPM trap. Typically used when leaving
5988 * VT-x to execute any instruction.
5989 *
5990 * @param pvCpu Pointer to the VMCPU.
5991 */
5992static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
5993{
5994 Assert(pVCpu->hm.s.Event.fPending);
5995
5996 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5997 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5998 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5999 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6000
6001 /* If a trap was already pending, we did something wrong! */
6002 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6003
6004 TRPMEVENT enmTrapType;
6005 switch (uVectorType)
6006 {
6007 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6008 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6009 enmTrapType = TRPM_HARDWARE_INT;
6010 break;
6011
6012 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6013 enmTrapType = TRPM_SOFTWARE_INT;
6014 break;
6015
6016 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6017 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6018 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6019 enmTrapType = TRPM_TRAP;
6020 break;
6021
6022 default:
6023 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6024 enmTrapType = TRPM_32BIT_HACK;
6025 break;
6026 }
6027
6028 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6029
6030 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6031 AssertRC(rc);
6032
6033 if (fErrorCodeValid)
6034 TRPMSetErrorCode(pVCpu, uErrorCode);
6035
6036 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6037 && uVector == X86_XCPT_PF)
6038 {
6039 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6040 }
6041 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6042 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6043 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6044 {
6045 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6046 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6047 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6048 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6049 }
6050 pVCpu->hm.s.Event.fPending = false;
6051}
6052
6053
6054/**
6055 * Does the necessary state syncing before returning to ring-3 for any reason
6056 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6057 *
6058 * @param pVM Pointer to the VM.
6059 * @param pVCpu Pointer to the VMCPU.
6060 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6061 * out-of-sync. Make sure to update the required fields
6062 * before using them.
6063 *
6064 * @remarks No-long-jmp zone!!!
6065 */
6066static void hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6067{
6068 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6069 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6070
6071 /* Avoid repeating this work when thread-context hooks are used and we had been preempted before
6072 which would've done this work from the VMXR0ThreadCtxCallback(). */
6073 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
6074 bool fPreemptDisabled = false;
6075 if (RTThreadPreemptIsEnabled(NIL_RTTHREAD))
6076 {
6077 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
6078 RTThreadPreemptDisable(&PreemptState);
6079 fPreemptDisabled = true;
6080 }
6081
6082 if (!pVCpu->hm.s.fLeaveDone)
6083 {
6084 Log4Func(("HostCpuId=%u\n", RTMpCpuId()));
6085
6086 /* Save the guest state if necessary. */
6087 if (pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6088 {
6089 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6090 AssertRC(rc);
6091 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6092 }
6093
6094 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6095 if (CPUMIsGuestFPUStateActive(pVCpu))
6096 {
6097 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6098 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6099 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
6100 }
6101
6102 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6103#ifdef VBOX_STRICT
6104 if (CPUMIsHyperDebugStateActive(pVCpu))
6105 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6106#endif
6107 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6108 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
6109 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6110 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6111
6112 /* Restore host-state bits that VT-x only restores partially. */
6113 if (pVCpu->hm.s.vmx.fRestoreHostFlags)
6114 {
6115 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6116 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6117 }
6118
6119 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6120 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6121 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6122 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6123 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6124 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6125 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6126 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6127
6128 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6129
6130 /** @todo This kinda defeats the purpose of having preemption hooks.
6131 * The problem is, deregistering the hooks should be moved to a place that
6132 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6133 * context.
6134 */
6135 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6136 {
6137 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6138 AssertRC(rc);
6139 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6140 }
6141 pVCpu->hm.s.vmx.uVmcsState &= ~HMVMX_VMCS_STATE_LAUNCHED;
6142 pVCpu->hm.s.fLeaveDone = true;
6143 }
6144
6145 /* Restore preemption if we previous disabled it ourselves. */
6146 if (fPreemptDisabled)
6147 RTThreadPreemptRestore(&PreemptState);
6148}
6149
6150
6151/**
6152 * Does the necessary state syncing before doing a longjmp to ring-3.
6153 *
6154 * @param pVM Pointer to the VM.
6155 * @param pVCpu Pointer to the VMCPU.
6156 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6157 * out-of-sync. Make sure to update the required fields
6158 * before using them.
6159 *
6160 * @remarks No-long-jmp zone!!!
6161 */
6162DECLINLINE(void) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6163{
6164 hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6165}
6166
6167
6168/**
6169 * Take necessary actions before going back to ring-3.
6170 *
6171 * An action requires us to go back to ring-3. This function does the necessary
6172 * steps before we can safely return to ring-3. This is not the same as longjmps
6173 * to ring-3, this is voluntary and prepares the guest so it may continue
6174 * executing outside HM (recompiler/IEM).
6175 *
6176 * @param pVM Pointer to the VM.
6177 * @param pVCpu Pointer to the VMCPU.
6178 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6179 * out-of-sync. Make sure to update the required fields
6180 * before using them.
6181 * @param rcExit The reason for exiting to ring-3. Can be
6182 * VINF_VMM_UNKNOWN_RING3_CALL.
6183 */
6184static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6185{
6186 Assert(pVM);
6187 Assert(pVCpu);
6188 Assert(pMixedCtx);
6189 HMVMX_ASSERT_PREEMPT_SAFE();
6190
6191 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6192 {
6193 /* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
6194 return;
6195 }
6196 else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6197 {
6198 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6199 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6200 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6201 pVCpu->hm.s.vmx.LastError.idCurrentCpu = RTMpCpuId();
6202 return;
6203 }
6204
6205 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6206 VMMRZCallRing3Disable(pVCpu);
6207 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6208
6209 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6210 if (pVCpu->hm.s.Event.fPending)
6211 {
6212 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6213 Assert(!pVCpu->hm.s.Event.fPending);
6214 }
6215
6216 /* Save guest state and restore host state bits. */
6217 hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6218 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6219
6220 /* Sync recompiler state. */
6221 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6222 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6223 | CPUM_CHANGED_LDTR
6224 | CPUM_CHANGED_GDTR
6225 | CPUM_CHANGED_IDTR
6226 | CPUM_CHANGED_TR
6227 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6228 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6229 if ( pVM->hm.s.fNestedPaging
6230 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6231 {
6232 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6233 }
6234
6235 /*
6236 * Clear the X86_EFL_TF if necessary .
6237 */
6238 if (pVCpu->hm.s.fClearTrapFlag)
6239 {
6240 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6241 pMixedCtx->eflags.Bits.u1TF = 0;
6242 pVCpu->hm.s.fClearTrapFlag = false;
6243 }
6244 /** @todo there seems to be issues with the resume flag when the monitor trap
6245 * flag is pending without being used. Seen early in bios init when
6246 * accessing APIC page in prot mode. */
6247
6248 /* On our way back from ring-3 the following needs to be done. */
6249 if (rcExit == VINF_EM_RAW_INTERRUPT)
6250 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT;
6251 else
6252 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_ALL_GUEST;
6253
6254 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6255 VMMRZCallRing3Enable(pVCpu);
6256}
6257
6258
6259/**
6260 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6261 * longjump to ring-3 and possibly get preempted.
6262 *
6263 * @param pVCpu Pointer to the VMCPU.
6264 * @param enmOperation The operation causing the ring-3 longjump.
6265 * @param pvUser The user argument (pointer to the possibly
6266 * out-of-date guest-CPU context).
6267 *
6268 * @remarks Must never be called with @a enmOperation ==
6269 * VMMCALLRING3_VM_R0_ASSERTION.
6270 */
6271DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6272{
6273 /* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6274 Assert(pVCpu);
6275 Assert(pvUser);
6276 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6277 HMVMX_ASSERT_PREEMPT_SAFE();
6278
6279 VMMRZCallRing3Disable(pVCpu);
6280 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6281
6282 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6283 hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6284
6285 VMMRZCallRing3Enable(pVCpu);
6286}
6287
6288
6289/**
6290 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6291 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6292 *
6293 * @param pVCpu Pointer to the VMCPU.
6294 */
6295DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6296{
6297 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6298 {
6299 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6300 {
6301 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6302 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6303 AssertRC(rc);
6304 }
6305 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6306}
6307
6308
6309/**
6310 * Injects any pending events into the guest if the guest is in a state to
6311 * receive them.
6312 *
6313 * @returns VBox status code (informational status codes included).
6314 * @param pVCpu Pointer to the VMCPU.
6315 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6316 * out-of-sync. Make sure to update the required fields
6317 * before using them.
6318 */
6319static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6320{
6321 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6322 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6323 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6324 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6325
6326 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6327 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6328 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6329 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6330 Assert(!TRPMHasTrap(pVCpu));
6331
6332 int rc = VINF_SUCCESS;
6333 if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
6334 {
6335 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6336 bool fInject = true;
6337 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6338 {
6339 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6340 AssertRCReturn(rc, rc);
6341 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6342 if ( fBlockInt
6343 || fBlockSti
6344 || fBlockMovSS)
6345 {
6346 fInject = false;
6347 }
6348 }
6349 else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6350 && ( fBlockMovSS
6351 || fBlockSti))
6352 {
6353 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6354 fInject = false;
6355 }
6356
6357 if (fInject)
6358 {
6359 Log4(("Injecting pending event vcpu[%RU32]\n", pVCpu->idCpu));
6360 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6361 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6362 AssertRCReturn(rc, rc);
6363 pVCpu->hm.s.Event.fPending = false;
6364
6365#ifdef VBOX_WITH_STATISTICS
6366 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6367 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6368 else
6369 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6370#endif
6371 }
6372 else
6373 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6374 } /** @todo SMI. SMIs take priority over NMIs. */
6375 else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6376 {
6377 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6378 if ( !fBlockMovSS
6379 && !fBlockSti)
6380 {
6381 Log4(("Injecting NMI\n"));
6382 uint32_t u32IntrInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6383 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6384 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */,
6385 0 /* GCPtrFaultAddress */, &uIntrState);
6386 AssertRCReturn(rc, rc);
6387 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6388
6389 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6390 }
6391 else
6392 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6393 }
6394 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6395 && !pVCpu->hm.s.fSingleInstruction)
6396 {
6397 /* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
6398 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6399 AssertRCReturn(rc, rc);
6400 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6401 if ( !fBlockInt
6402 && !fBlockSti
6403 && !fBlockMovSS)
6404 {
6405 uint8_t u8Interrupt;
6406 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6407 if (RT_SUCCESS(rc))
6408 {
6409 Log4(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
6410 uint32_t u32IntrInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6411 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6412 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */,
6413 0 /* GCPtrFaultAddress */, &uIntrState);
6414
6415 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6416 }
6417 else
6418 {
6419 /** @todo Does this actually happen? If not turn it into an assertion. */
6420 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6421 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6422 rc = VINF_SUCCESS;
6423 }
6424 }
6425 else
6426 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6427 }
6428
6429 /*
6430 * Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6431 * hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6432 */
6433 fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6434 fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6435 int rc2 = VINF_SUCCESS;
6436 if ( fBlockSti
6437 || fBlockMovSS)
6438 {
6439 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6440 {
6441 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6442 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6443 {
6444 /*
6445 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6446 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6447 * See Intel spec. 27.3.4 "Saving Non-Register State".
6448 */
6449 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6450 AssertRCReturn(rc, rc);
6451 }
6452 }
6453 else
6454 {
6455 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6456 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6457 uIntrState = 0;
6458 }
6459 }
6460
6461 /*
6462 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6463 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6464 */
6465 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6466 AssertRC(rc2);
6467
6468 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6469 return rc;
6470}
6471
6472
6473/**
6474 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6475 *
6476 * @param pVCpu Pointer to the VMCPU.
6477 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6478 * out-of-sync. Make sure to update the required fields
6479 * before using them.
6480 */
6481DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6482{
6483 uint32_t u32IntrInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6484 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6485}
6486
6487
6488/**
6489 * Injects a double-fault (#DF) exception into the VM.
6490 *
6491 * @returns VBox status code (informational status code included).
6492 * @param pVCpu Pointer to the VMCPU.
6493 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6494 * out-of-sync. Make sure to update the required fields
6495 * before using them.
6496 */
6497DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6498{
6499 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6500 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6501 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6502 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6503 puIntrState);
6504}
6505
6506
6507/**
6508 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6509 *
6510 * @param pVCpu Pointer to the VMCPU.
6511 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6512 * out-of-sync. Make sure to update the required fields
6513 * before using them.
6514 */
6515DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6516{
6517 uint32_t u32IntrInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6518 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6519 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6520}
6521
6522
6523/**
6524 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6525 *
6526 * @param pVCpu Pointer to the VMCPU.
6527 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6528 * out-of-sync. Make sure to update the required fields
6529 * before using them.
6530 * @param cbInstr The value of RIP that is to be pushed on the guest
6531 * stack.
6532 */
6533DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6534{
6535 uint32_t u32IntrInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6536 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6537 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6538}
6539
6540
6541/**
6542 * Injects a general-protection (#GP) fault into the VM.
6543 *
6544 * @returns VBox status code (informational status code included).
6545 * @param pVCpu Pointer to the VMCPU.
6546 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6547 * out-of-sync. Make sure to update the required fields
6548 * before using them.
6549 * @param u32ErrorCode The error code associated with the #GP.
6550 */
6551DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6552 uint32_t *puIntrState)
6553{
6554 uint32_t u32IntrInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6555 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6556 if (fErrorCodeValid)
6557 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6558 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6559 puIntrState);
6560}
6561
6562
6563/**
6564 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6565 *
6566 * @param pVCpu Pointer to the VMCPU.
6567 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6568 * out-of-sync. Make sure to update the required fields
6569 * before using them.
6570 * @param uVector The software interrupt vector number.
6571 * @param cbInstr The value of RIP that is to be pushed on the guest
6572 * stack.
6573 */
6574DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6575{
6576 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6577 if ( uVector == X86_XCPT_BP
6578 || uVector == X86_XCPT_OF)
6579 {
6580 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6581 }
6582 else
6583 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6584 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6585}
6586
6587
6588/**
6589 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6590 * stack.
6591 *
6592 * @returns VBox status code (information status code included).
6593 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6594 * @param pVM Pointer to the VM.
6595 * @param pMixedCtx Pointer to the guest-CPU context.
6596 * @param uValue The value to push to the guest stack.
6597 */
6598DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6599{
6600 /*
6601 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6602 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6603 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6604 */
6605 if (pMixedCtx->sp == 1)
6606 return VINF_EM_RESET;
6607 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6608 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6609 AssertRCReturn(rc, rc);
6610 return rc;
6611}
6612
6613
6614/**
6615 * Injects an event into the guest upon VM-entry by updating the relevant fields
6616 * in the VM-entry area in the VMCS.
6617 *
6618 * @returns VBox status code (informational error codes included).
6619 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6620 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6621 *
6622 * @param pVCpu Pointer to the VMCPU.
6623 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6624 * be out-of-sync. Make sure to update the required
6625 * fields before using them.
6626 * @param u64IntrInfo The VM-entry interruption-information field.
6627 * @param cbInstr The VM-entry instruction length in bytes (for
6628 * software interrupts, exceptions and privileged
6629 * software exceptions).
6630 * @param u32ErrCode The VM-entry exception error code.
6631 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6632 * @param puIntrState Pointer to the current guest interruptibility-state.
6633 * This interruptibility-state will be updated if
6634 * necessary. This cannot not be NULL.
6635 *
6636 * @remarks No-long-jump zone!!!
6637 * @remarks Requires CR0!
6638 */
6639static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6640 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6641{
6642 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6643 AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6644 Assert(puIntrState);
6645 uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6646
6647 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6648 const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6649
6650#ifdef VBOX_STRICT
6651 /* Validate the error-code-valid bit for hardware exceptions. */
6652 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6653 {
6654 switch (uVector)
6655 {
6656 case X86_XCPT_PF:
6657 case X86_XCPT_DF:
6658 case X86_XCPT_TS:
6659 case X86_XCPT_NP:
6660 case X86_XCPT_SS:
6661 case X86_XCPT_GP:
6662 case X86_XCPT_AC:
6663 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6664 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6665 /* fallthru */
6666 default:
6667 break;
6668 }
6669 }
6670#endif
6671
6672 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
6673 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6674 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6675
6676 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6677
6678 /* We require CR0 to check if the guest is in real-mode. */
6679 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6680 AssertRCReturn(rc, rc);
6681
6682 /*
6683 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6684 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6685 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6686 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6687 */
6688 if (CPUMIsGuestInRealModeEx(pMixedCtx))
6689 {
6690 PVM pVM = pVCpu->CTX_SUFF(pVM);
6691 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6692 {
6693 Assert(PDMVmmDevHeapIsEnabled(pVM));
6694 Assert(pVM->hm.s.vmx.pRealModeTSS);
6695
6696 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6697 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6698 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6699 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6700 AssertRCReturn(rc, rc);
6701 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6702
6703 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6704 const size_t cbIdtEntry = 4;
6705 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6706 {
6707 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6708 if (uVector == X86_XCPT_DF)
6709 return VINF_EM_RESET;
6710 else if (uVector == X86_XCPT_GP)
6711 {
6712 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6713 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6714 }
6715
6716 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6717 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6718 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
6719 }
6720
6721 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6722 uint16_t uGuestIp = pMixedCtx->ip;
6723 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6724 {
6725 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
6726 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
6727 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6728 }
6729 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6730 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6731
6732 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6733 uint16_t offIdtEntry = 0;
6734 RTSEL selIdtEntry = 0;
6735 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6736 rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6737 rc |= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6738 AssertRCReturn(rc, rc);
6739
6740 /* Construct the stack frame for the interrupt/exception handler. */
6741 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6742 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6743 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6744 AssertRCReturn(rc, rc);
6745
6746 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
6747 if (rc == VINF_SUCCESS)
6748 {
6749 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
6750 pMixedCtx->rip = offIdtEntry;
6751 pMixedCtx->cs.Sel = selIdtEntry;
6752 pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6753 if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6754 && uVector == X86_XCPT_PF)
6755 {
6756 pMixedCtx->cr2 = GCPtrFaultAddress;
6757 }
6758 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS
6759 | HM_CHANGED_GUEST_RIP
6760 | HM_CHANGED_GUEST_RFLAGS
6761 | HM_CHANGED_GUEST_RSP;
6762
6763 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6764 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6765 {
6766 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6767 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6768 Log4(("Clearing inhibition due to STI.\n"));
6769 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6770 }
6771 Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6772 }
6773 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6774 return rc;
6775 }
6776 else
6777 {
6778 /*
6779 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6780 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6781 */
6782 u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6783 }
6784 }
6785
6786 /* Validate. */
6787 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6788 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6789 Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6790
6791 /* Inject. */
6792 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6793 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6794 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6795 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6796
6797 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6798 && uVector == X86_XCPT_PF)
6799 {
6800 pMixedCtx->cr2 = GCPtrFaultAddress;
6801 }
6802
6803 Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6804 u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6805
6806 AssertRCReturn(rc, rc);
6807 return rc;
6808}
6809
6810
6811/**
6812 * Enters the VT-x session.
6813 *
6814 * @returns VBox status code.
6815 * @param pVM Pointer to the VM.
6816 * @param pVCpu Pointer to the VMCPU.
6817 * @param pCpu Pointer to the CPU info struct.
6818 */
6819VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
6820{
6821 AssertPtr(pVM);
6822 AssertPtr(pVCpu);
6823 Assert(pVM->hm.s.vmx.fSupported);
6824 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6825 NOREF(pCpu);
6826
6827 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6828
6829#ifdef VBOX_STRICT
6830 /* Make sure we're in VMX root mode. */
6831 RTCCUINTREG u32HostCR4 = ASMGetCR4();
6832 if (!(u32HostCR4 & X86_CR4_VMXE))
6833 {
6834 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6835 return VERR_VMX_X86_CR4_VMXE_CLEARED;
6836 }
6837#endif
6838
6839 /*
6840 * The VMCS state here will not be reliable because we deregister the hook in VMMR0EntryFast()
6841 * on the way out. If we had got a preempt/resume callback -after- hmR0VmxLeave() but before
6842 * deregistering the hook, the VMCS state will be ACTIVE. Once deregistered we no longer get
6843 * notifications and lose track. Following that if we get rescheduled to another host CPU, the
6844 * VMCS state says ACTIVE even though it really is not.
6845 *
6846 * Load the VCPU's VMCS as the current (and active) one.
6847 */
6848 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6849 if (RT_FAILURE(rc))
6850 return rc;
6851 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
6852
6853 pVCpu->hm.s.fLeaveDone = false;
6854 return VINF_SUCCESS;
6855}
6856
6857
6858/**
6859 * The thread-context callback (only on platforms which support it).
6860 *
6861 * @param enmEvent The thread-context event.
6862 * @param pVCpu Pointer to the VMCPU.
6863 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
6864 * @thread EMT.
6865 */
6866VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
6867{
6868 switch (enmEvent)
6869 {
6870 case RTTHREADCTXEVENT_PREEMPTING:
6871 {
6872 /** @todo Stats. */
6873 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6874 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
6875 VMCPU_ASSERT_EMT(pVCpu);
6876
6877 PVM pVM = pVCpu->CTX_SUFF(pVM);
6878 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
6879
6880 /* No longjmps (logger flushes, locks) in this fragile context. */
6881 VMMRZCallRing3Disable(pVCpu);
6882 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
6883
6884 /* Save the guest-state, restore host-state (FPU, debug etc.). */
6885 hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6886
6887 int rc;
6888#if 0
6889 /* Flush VMCS CPU state to the VMCS region in memory. */
6890 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6891 {
6892 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6893 AssertRC(rc);
6894 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6895 }
6896 pVCpu->hm.s.vmx.uVmcsState &= ~HMVMX_VMCS_STATE_LAUNCHED;
6897#endif
6898
6899 /* Leave HM context, takes care of local init (term). */
6900 rc = HMR0LeaveEx(pVCpu);
6901 AssertRC(rc);
6902
6903 /* Restore longjmp state. */
6904 VMMRZCallRing3Enable(pVCpu);
6905 NOREF(rc);
6906 break;
6907 }
6908
6909 case RTTHREADCTXEVENT_RESUMED:
6910 {
6911 /** @todo Stats. */
6912 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6913 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
6914 VMCPU_ASSERT_EMT(pVCpu);
6915
6916 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
6917 VMMRZCallRing3Disable(pVCpu);
6918 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
6919
6920 /* Initialize the bare minimum state required for HM. This takes care of
6921 initializing VT-x if necessary (onlined CPUs, local init etc.) */
6922 HMR0EnterEx(pVCpu);
6923 Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_GUEST_CR0));
6924
6925 /* Load the active VMCS as the current one. */
6926 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
6927 {
6928 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6929 AssertRC(rc); NOREF(rc);
6930 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
6931 Log4Func(("Activated: HostCpuId=%u\n", RTMpCpuId()));
6932 }
6933 pVCpu->hm.s.fLeaveDone = false;
6934 VMMRZCallRing3Enable(pVCpu);
6935 break;
6936 }
6937
6938 default:
6939 break;
6940 }
6941}
6942
6943
6944/**
6945 * Leaves the VT-x session.
6946 *
6947 * @returns VBox status code.
6948 * @param pVM Pointer to the VM.
6949 * @param pVCpu Pointer to the VMCPU.
6950 * @param pCtx Pointer to the guest-CPU context.
6951 */
6952VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6953{
6954 NOREF(pVCpu);
6955 NOREF(pVM);
6956 NOREF(pCtx);
6957
6958 /* Everything is taken care of in hmR0VmxLeave() and VMXR0ThreadCtxCallback()'s preempt event. */
6959 return VINF_SUCCESS;
6960}
6961
6962
6963/**
6964 * Saves the host state in the VMCS host-state.
6965 * Sets up the VM-exit MSR-load area.
6966 *
6967 * The CPU state will be loaded from these fields on every successful VM-exit.
6968 *
6969 * @returns VBox status code.
6970 * @param pVM Pointer to the VM.
6971 * @param pVCpu Pointer to the VMCPU.
6972 *
6973 * @remarks No-long-jump zone!!!
6974 */
6975static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
6976{
6977 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6978
6979 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6980 return VINF_SUCCESS;
6981
6982 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6983 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6984
6985 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6986 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6987
6988 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6989 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6990
6991 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6992 return rc;
6993}
6994
6995
6996/**
6997 * Saves the host state in the VMCS host-state.
6998 *
6999 * @returns VBox status code.
7000 * @param pVM Pointer to the VM.
7001 * @param pVCpu Pointer to the VMCPU.
7002 *
7003 * @remarks No-long-jump zone!!!
7004 */
7005VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7006{
7007 AssertPtr(pVM);
7008 AssertPtr(pVCpu);
7009
7010 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7011 return hmR0VmxSaveHostState(pVM, pVCpu);
7012}
7013
7014
7015/**
7016 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7017 * loaded from these fields on every successful VM-entry.
7018 *
7019 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7020 * Sets up the VM-entry controls.
7021 * Sets up the appropriate VMX non-root function to execute guest code based on
7022 * the guest CPU mode.
7023 *
7024 * @returns VBox status code.
7025 * @param pVM Pointer to the VM.
7026 * @param pVCpu Pointer to the VMCPU.
7027 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7028 * out-of-sync. Make sure to update the required fields
7029 * before using them.
7030 *
7031 * @remarks No-long-jump zone!!!
7032 */
7033static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7034{
7035 AssertPtr(pVM);
7036 AssertPtr(pVCpu);
7037 AssertPtr(pMixedCtx);
7038 HMVMX_ASSERT_PREEMPT_SAFE();
7039
7040#ifdef LOG_ENABLED
7041 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7042 * probably not initialized yet? Anyway this will do for now.
7043 *
7044 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7045 * interface and disable ring-3 calls when thread-context hooks are not
7046 * available. */
7047 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7048 VMMR0LogFlushDisable(pVCpu);
7049#endif
7050
7051 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7052
7053 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7054
7055 /* Determine real-on-v86 mode. */
7056 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7057 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7058 && CPUMIsGuestInRealModeEx(pMixedCtx))
7059 {
7060 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7061 }
7062
7063 /*
7064 * Load the guest-state into the VMCS.
7065 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7066 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7067 */
7068 int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7069 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7070
7071 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7072 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7073
7074 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7075 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7076
7077 rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
7078 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7079
7080 /* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
7081 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7082 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7083
7084 rc = hmR0VmxLoadGuestDebugState(pVCpu, pMixedCtx);
7085 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugState: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7086
7087 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7088 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7089
7090 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7091 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7092
7093 /* Must be done after hmR0VmxLoadGuestDebugState() as it may have updated eflags.TF for debugging purposes. */
7094 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7095 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7096
7097 rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7098 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7099
7100 /* Clear any unused and reserved bits. */
7101 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
7102
7103 AssertMsg(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST),
7104 ("Missed updating flags while loading guest state. pVM=%p pVCpu=%p idCpu=%RU32 fContextUseFlags=%#RX32\n",
7105 pVM, pVCpu, pVCpu->idCpu, pVCpu->hm.s.fContextUseFlags));
7106
7107#ifdef LOG_ENABLED
7108 /* Only reenable log-flushing if the caller has it enabled. */
7109 if (!fCallerDisabledLogFlush)
7110 VMMR0LogFlushEnable(pVCpu);
7111#endif
7112
7113 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7114 return rc;
7115}
7116
7117
7118/**
7119 * Loads the guest state into the VMCS guest-state area.
7120 *
7121 * @returns VBox status code.
7122 * @param pVM Pointer to the VM.
7123 * @param pVCpu Pointer to the VMCPU.
7124 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7125 * out-of-sync. Make sure to update the required fields
7126 * before using them.
7127 *
7128 * @remarks No-long-jump zone!!!
7129 */
7130VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7131{
7132 /*
7133 * Avoid reloading the guest state on longjmp reentrants and do it lazily just before executing the guest.
7134 * When thread-context hooks are not used: This only helps when we get rescheduled more than once to a
7135 * different host CPU on a longjmp trip before finally executing guest code.
7136 *
7137 * When thread-context hooks are used: We avoid loading the guest state here for the above reason plus
7138 * we can avoid doing it while preemption is disabled (which it is here).
7139 */
7140 return VINF_SUCCESS;
7141}
7142
7143/**
7144 * Wrapper for loading the guest-state bits in the inner VT-x execution loop.
7145 *
7146 * @param pVM Pointer to the VM.
7147 * @param pVCpu Pointer to the VMCPU.
7148 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7149 * out-of-sync. Make sure to update the required fields
7150 * before using them.
7151 */
7152DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7153{
7154 Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7155#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7156 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
7157#endif
7158
7159 if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7160 {
7161 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7162 AssertRC(rc);
7163 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7164 }
7165 else if (pVCpu->hm.s.fContextUseFlags)
7166 {
7167 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7168 AssertRC(rc);
7169 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7170 }
7171
7172 /* When thread-context hooks are available, we could be preempted which means re-updating Guest.CR0
7173 (shared FPU state) and debug controls (shared debug state). This is done in hmR0VmxPreRunGuestCommitted() */
7174#ifdef VBOX_STRICT
7175 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7176 {
7177 AssertMsg( !(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST)
7178 || (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST) == HM_CHANGED_GUEST_CR0
7179 || (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST) == (HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG),
7180 ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7181 }
7182 else
7183 {
7184 AssertMsg(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST), ("fContextUseFlags=%#x\n",
7185 pVCpu->hm.s.fContextUseFlags));
7186 }
7187#endif
7188
7189#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7190 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7191 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7192 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7193#endif
7194}
7195
7196
7197/**
7198 * Does the preparations before executing guest code in VT-x.
7199 *
7200 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7201 * recompiler. We must be cautious what we do here regarding committing
7202 * guest-state information into the VMCS assuming we assuredly execute the
7203 * guest in VT-x. If we fall back to the recompiler after updating the VMCS and
7204 * clearing the common-state (TRPM/forceflags), we must undo those changes so
7205 * that the recompiler can (and should) use them when it resumes guest
7206 * execution. Otherwise such operations must be done when we can no longer
7207 * exit to ring-3.
7208 *
7209 * @returns VBox status code (informational status codes included).
7210 * @retval VINF_SUCCESS if we can proceed with running the guest.
7211 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
7212 * into the guest.
7213 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7214 *
7215 * @param pVM Pointer to the VM.
7216 * @param pVCpu Pointer to the VMCPU.
7217 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7218 * out-of-sync. Make sure to update the required fields
7219 * before using them.
7220 * @param pVmxTransient Pointer to the VMX transient structure.
7221 *
7222 * @remarks Called with preemption disabled.
7223 */
7224static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7225{
7226 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7227
7228#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7229 PGMRZDynMapFlushAutoSet(pVCpu);
7230#endif
7231
7232 /* Check force flag actions that might require us to go back to ring-3. */
7233 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7234 if (rc != VINF_SUCCESS)
7235 return rc;
7236
7237#ifndef IEM_VERIFICATION_MODE_FULL
7238 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7239 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7240 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7241 {
7242 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7243 RTGCPHYS GCPhysApicBase;
7244 GCPhysApicBase = pMixedCtx->msrApicBase;
7245 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7246
7247 /* Unalias any existing mapping. */
7248 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7249 AssertRCReturn(rc, rc);
7250
7251 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7252 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7253 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7254 AssertRCReturn(rc, rc);
7255
7256 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7257 }
7258#endif /* !IEM_VERIFICATION_MODE_FULL */
7259
7260 /*
7261 * When thread-context hooks are used, load the required guest-state bits
7262 * here before we go ahead and disable interrupts.
7263 */
7264 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7265 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7266
7267#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7268 /* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
7269 pVmxTransient->uEflags = ASMIntDisableFlags();
7270 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7271 {
7272 ASMSetFlags(pVmxTransient->uEflags);
7273 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7274 /* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
7275 return VINF_EM_RAW_INTERRUPT;
7276 }
7277 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7278 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
7279#endif
7280
7281 /*
7282 * Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
7283 * state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
7284 * interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
7285 */
7286 /** @todo Rework event evaluation and injection to be completely separate.
7287 * Update: Tried it, problem with handling halts. Control never returns to VT-x
7288 * if we exit VT-x with external interrupt pending in a TRPM event. */
7289 if (TRPMHasTrap(pVCpu))
7290 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7291
7292 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7293 AssertRCReturn(rc, rc);
7294 return rc;
7295}
7296
7297
7298/**
7299 * Prepares to run guest code in VT-x and we've committed to doing so. This
7300 * means there is no backing out to ring-3 or anywhere else at this
7301 * point.
7302 *
7303 * @param pVM Pointer to the VM.
7304 * @param pVCpu Pointer to the VMCPU.
7305 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7306 * out-of-sync. Make sure to update the required fields
7307 * before using them.
7308 * @param pVmxTransient Pointer to the VMX transient structure.
7309 *
7310 * @remarks Called with preemption disabled.
7311 * @remarks No-long-jump zone!!!
7312 */
7313static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7314{
7315 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7316 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7317
7318#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7319 /** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
7320 pVmxTransient->uEflags = ASMIntDisableFlags();
7321 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
7322#endif
7323
7324 /*
7325 * Load the host state bits as we may've been preempted
7326 * (only happens when thread-context hooks are used).
7327 */
7328 int rc = VINF_SUCCESS;
7329 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
7330 {
7331 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7332 rc = hmR0VmxSaveHostState(pVM, pVCpu);
7333 AssertRC(rc);
7334 }
7335 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7336
7337 /*
7338 * When thread-context hooks are not used we need to load the required
7339 * guest state bits here i.e. when we can no longer be preempted.
7340 */
7341 if (!VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7342 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7343 else
7344 {
7345 /*
7346 * If we got preempted previously while loading the guest state, the guest FPU and debug
7347 * state need to be re-updated because we share them with the host state.
7348 */
7349 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7350 {
7351 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
7352 hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
7353 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
7354 hmR0VmxLoadGuestDebugState(pVCpu, pMixedCtx);
7355 }
7356 else
7357 {
7358 /*
7359 * If we are injecting events real-on-v86 mode guest then we potentially have to update
7360 * RIP and other registers. Just reload the state here if we're in real-on-v86 mode.
7361 */
7362 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7363 }
7364 }
7365 AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7366
7367 /*
7368 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7369 */
7370 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7371 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7372
7373 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7374 || HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
7375 {
7376 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7377 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7378 }
7379
7380 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7381 hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
7382
7383 RTCPUID idCurrentCpu = HMR0GetCurrentCpu()->idCpu;
7384 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7385 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7386
7387 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7388
7389 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7390 to start executing. */
7391
7392#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7393 /*
7394 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7395 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7396 */
7397 if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7398 && !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7399 {
7400 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7401 uint64_t u64HostTscAux = 0;
7402 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7403 AssertRC(rc2);
7404 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7405 }
7406#endif
7407}
7408
7409
7410/**
7411 * Performs some essential restoration of state after running guest code in
7412 * VT-x.
7413 *
7414 * @param pVM Pointer to the VM.
7415 * @param pVCpu Pointer to the VMCPU.
7416 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7417 * out-of-sync. Make sure to update the required fields
7418 * before using them.
7419 * @param pVmxTransient Pointer to the VMX transient structure.
7420 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7421 *
7422 * @remarks Called with interrupts disabled.
7423 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7424 * unconditionally when it is safe to do so.
7425 */
7426static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7427{
7428 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7429
7430 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7431 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7432 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7433 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7434 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7435
7436 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7437 {
7438#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7439 /* Restore host's TSC_AUX. */
7440 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7441 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7442#endif
7443 /** @todo Find a way to fix hardcoding a guestimate. */
7444 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7445 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7446 }
7447
7448 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7449 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7450 Assert(!(ASMGetFlags() & X86_EFL_IF));
7451 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7452
7453 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7454 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7455
7456 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7457 uint32_t uExitReason;
7458 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7459 rc |= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7460 AssertRC(rc);
7461 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7462 pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7463
7464 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
7465 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7466
7467 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7468 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7469 {
7470 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7471 pVmxTransient->fVMEntryFailed));
7472 return;
7473 }
7474
7475 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7476 {
7477 /* Update the guest interruptibility-state from the VMCS. */
7478 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7479#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7480 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7481 AssertRC(rc);
7482#endif
7483 /*
7484 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7485 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7486 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7487 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7488 */
7489 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7490 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7491 {
7492 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7493 AssertRC(rc);
7494 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
7495 }
7496 }
7497}
7498
7499
7500
7501/**
7502 * Runs the guest code using VT-x the normal way.
7503 *
7504 * @returns VBox status code.
7505 * @param pVM Pointer to the VM.
7506 * @param pVCpu Pointer to the VMCPU.
7507 * @param pCtx Pointer to the guest-CPU context.
7508 *
7509 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
7510 * @remarks Called with preemption disabled.
7511 */
7512static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7513{
7514 VMXTRANSIENT VmxTransient;
7515 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7516 int rc = VERR_INTERNAL_ERROR_5;
7517 uint32_t cLoops = 0;
7518
7519 for (;; cLoops++)
7520 {
7521 Assert(!HMR0SuspendPending());
7522 HMVMX_ASSERT_CPU_SAFE();
7523
7524 /* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7525 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7526 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7527 if (rc != VINF_SUCCESS)
7528 break;
7529
7530 /*
7531 * No longjmps to ring-3 from this point on!!!
7532 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7533 * This also disables flushing of the R0-logger instance (if any).
7534 */
7535 VMMRZCallRing3Disable(pVCpu);
7536 VMMRZCallRing3RemoveNotification(pVCpu);
7537 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7538
7539 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7540 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7541
7542 /*
7543 * Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7544 * This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7545 */
7546 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7547 if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7548 {
7549 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7550 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7551 return rc;
7552 }
7553
7554 /* Handle the VM-exit. */
7555 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7556 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7557 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7558 HMVMX_START_EXIT_DISPATCH_PROF();
7559#ifdef HMVMX_USE_FUNCTION_TABLE
7560 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7561#else
7562 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7563#endif
7564 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7565 if (rc != VINF_SUCCESS)
7566 break;
7567 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7568 {
7569 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7570 rc = VINF_EM_RAW_INTERRUPT;
7571 break;
7572 }
7573 }
7574
7575 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7576 return rc;
7577}
7578
7579
7580/**
7581 * Single steps guest code using VT-x.
7582 *
7583 * @returns VBox status code.
7584 * @param pVM Pointer to the VM.
7585 * @param pVCpu Pointer to the VMCPU.
7586 * @param pCtx Pointer to the guest-CPU context.
7587 *
7588 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7589 * @remarks Called with preemption disabled.
7590 */
7591static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7592{
7593 VMXTRANSIENT VmxTransient;
7594 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7595 int rc = VERR_INTERNAL_ERROR_5;
7596 uint32_t cLoops = 0;
7597 uint16_t uCsStart = pCtx->cs.Sel;
7598 uint64_t uRipStart = pCtx->rip;
7599
7600 for (;; cLoops++)
7601 {
7602 Assert(!HMR0SuspendPending());
7603 HMVMX_ASSERT_CPU_SAFE();
7604
7605 /* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7606 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7607 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7608 if (rc != VINF_SUCCESS)
7609 break;
7610
7611 /*
7612 * No longjmps to ring-3 from this point on!!!
7613 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7614 * This also disables flushing of the R0-logger instance (if any).
7615 */
7616 VMMRZCallRing3Disable(pVCpu);
7617 VMMRZCallRing3RemoveNotification(pVCpu);
7618 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7619
7620 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7621 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7622
7623 /*
7624 * Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
7625 * This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
7626 */
7627 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7628 if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7629 {
7630 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7631 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7632 return rc;
7633 }
7634
7635 /* Handle the VM-exit. */
7636 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7637 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7638 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7639 HMVMX_START_EXIT_DISPATCH_PROF();
7640#ifdef HMVMX_USE_FUNCTION_TABLE
7641 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7642#else
7643 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7644#endif
7645 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7646 if (rc != VINF_SUCCESS)
7647 break;
7648 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7649 {
7650 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7651 rc = VINF_EM_RAW_INTERRUPT;
7652 break;
7653 }
7654
7655 /*
7656 * Did the RIP change, if so, consider it a single step.
7657 * Otherwise, make sure one of the TFs gets set.
7658 */
7659 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7660 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7661 AssertRCReturn(rc2, rc2);
7662 if ( pCtx->rip != uRipStart
7663 || pCtx->cs.Sel != uCsStart)
7664 {
7665 rc = VINF_EM_DBG_STEPPED;
7666 break;
7667 }
7668 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
7669 }
7670
7671 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7672 return rc;
7673}
7674
7675
7676/**
7677 * Runs the guest code using VT-x.
7678 *
7679 * @returns VBox status code.
7680 * @param pVM Pointer to the VM.
7681 * @param pVCpu Pointer to the VMCPU.
7682 * @param pCtx Pointer to the guest-CPU context.
7683 *
7684 * @remarks Called with preemption disabled.
7685 */
7686VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7687{
7688 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7689 HMVMX_ASSERT_PREEMPT_SAFE();
7690
7691 int rc;
7692 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7693 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7694 else
7695 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7696
7697 if (rc == VERR_EM_INTERPRETER)
7698 rc = VINF_EM_RAW_EMULATE_INSTR;
7699 else if (rc == VINF_EM_RESET)
7700 rc = VINF_EM_TRIPLE_FAULT;
7701 hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7702 return rc;
7703}
7704
7705
7706#ifndef HMVMX_USE_FUNCTION_TABLE
7707DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7708{
7709 int rc;
7710 switch (rcReason)
7711 {
7712 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7713 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7714 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7715 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7716 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7717 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7718 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7719 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7720 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7721 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7722 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7723 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7724 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7725 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7726 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7727 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7728 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7729 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7730 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7731 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7732 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7733 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7734 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7735 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7736 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7737 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7738 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7739 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7740 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7741 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7742 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7743 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7744 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7745
7746 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7747 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7748 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7749 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7750 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7751 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7752 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7753 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7754 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7755
7756 case VMX_EXIT_VMCALL:
7757 case VMX_EXIT_VMCLEAR:
7758 case VMX_EXIT_VMLAUNCH:
7759 case VMX_EXIT_VMPTRLD:
7760 case VMX_EXIT_VMPTRST:
7761 case VMX_EXIT_VMREAD:
7762 case VMX_EXIT_VMRESUME:
7763 case VMX_EXIT_VMWRITE:
7764 case VMX_EXIT_VMXOFF:
7765 case VMX_EXIT_VMXON:
7766 case VMX_EXIT_INVEPT:
7767 case VMX_EXIT_INVVPID:
7768 case VMX_EXIT_VMFUNC:
7769 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7770 break;
7771 default:
7772 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7773 break;
7774 }
7775 return rc;
7776}
7777#endif
7778
7779#ifdef DEBUG
7780/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7781# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7782 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7783
7784# define HMVMX_ASSERT_PREEMPT_CPUID() \
7785 do \
7786 { \
7787 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7788 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7789 } while (0)
7790
7791# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7792 do { \
7793 AssertPtr(pVCpu); \
7794 AssertPtr(pMixedCtx); \
7795 AssertPtr(pVmxTransient); \
7796 Assert(pVmxTransient->fVMEntryFailed == false); \
7797 Assert(ASMIntAreEnabled()); \
7798 HMVMX_ASSERT_PREEMPT_SAFE(); \
7799 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7800 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7801 HMVMX_ASSERT_PREEMPT_SAFE(); \
7802 if (VMMR0IsLogFlushDisabled(pVCpu)) \
7803 HMVMX_ASSERT_PREEMPT_CPUID(); \
7804 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7805 } while (0)
7806
7807# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7808 do { \
7809 Log4Func(("\n")); \
7810 } while(0)
7811#else /* Release builds */
7812# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7813# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7814#endif
7815
7816
7817/**
7818 * Advances the guest RIP after reading it from the VMCS.
7819 *
7820 * @returns VBox status code.
7821 * @param pVCpu Pointer to the VMCPU.
7822 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7823 * out-of-sync. Make sure to update the required fields
7824 * before using them.
7825 * @param pVmxTransient Pointer to the VMX transient structure.
7826 *
7827 * @remarks No-long-jump zone!!!
7828 */
7829DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7830{
7831 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7832 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7833 AssertRCReturn(rc, rc);
7834
7835 pMixedCtx->rip += pVmxTransient->cbInstr;
7836 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
7837 return rc;
7838}
7839
7840
7841/**
7842 * Tries to determine what part of the guest-state VT-x has deemed as invalid
7843 * and update error record fields accordingly.
7844 *
7845 * @return VMX_IGS_* return codes.
7846 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7847 * wrong with the guest state.
7848 *
7849 * @param pVM Pointer to the VM.
7850 * @param pVCpu Pointer to the VMCPU.
7851 * @param pCtx Pointer to the guest-CPU state.
7852 */
7853static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7854{
7855#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7856#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
7857 uError = (err); \
7858 break; \
7859 } else do {} while (0)
7860/* Duplicate of IEM_IS_CANONICAL(). */
7861#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
7862
7863 int rc;
7864 uint64_t u64Val;
7865 uint32_t u32Val;
7866 uint32_t uError = VMX_IGS_ERROR;
7867 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
7868
7869 do
7870 {
7871 /*
7872 * CR0.
7873 */
7874 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
7875 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 | pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
7876 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
7877 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
7878 if (fUnrestrictedGuest)
7879 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
7880
7881 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7882 AssertRCBreak(rc);
7883 HMVMX_CHECK_BREAK((u32Val & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
7884 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR0), VMX_IGS_CR0_FIXED0);
7885 if ( !fUnrestrictedGuest
7886 && (u32Val & X86_CR0_PG)
7887 && !(u32Val & X86_CR0_PE))
7888 {
7889 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
7890 }
7891
7892 /*
7893 * CR4.
7894 */
7895 uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
7896 uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 | pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
7897 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7898 AssertRCBreak(rc);
7899 HMVMX_CHECK_BREAK((u32Val & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
7900 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR4), VMX_IGS_CR4_FIXED0);
7901
7902 /*
7903 * IA32_DEBUGCTL MSR.
7904 */
7905 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
7906 AssertRCBreak(rc);
7907 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7908 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
7909 {
7910 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
7911 }
7912 uint64_t u64DebugCtlMsr = u64Val;
7913
7914#ifdef VBOX_STRICT
7915 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
7916 AssertRCBreak(rc);
7917 Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
7918#endif
7919 bool const fLongModeGuest = !!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
7920
7921 /*
7922 * RIP and RFLAGS.
7923 */
7924 uint32_t u32Eflags;
7925#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7926 if (HMVMX_IS_64BIT_HOST_MODE())
7927 {
7928 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
7929 AssertRCBreak(rc);
7930 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
7931 if ( !fLongModeGuest
7932 || !pCtx->cs.Attr.n.u1Long)
7933 {
7934 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
7935 }
7936 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
7937 * must be identical if the "IA32e mode guest" VM-entry control is 1
7938 * and CS.L is 1. No check applies if the CPU supports 64
7939 * linear-address bits. */
7940
7941 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
7942 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
7943 AssertRCBreak(rc);
7944 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
7945 VMX_IGS_RFLAGS_RESERVED);
7946 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7947 u32Eflags = u64Val;
7948 }
7949 else
7950#endif
7951 {
7952 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
7953 AssertRCBreak(rc);
7954 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
7955 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
7956 }
7957
7958 if ( fLongModeGuest
7959 || !(pCtx->cr0 & X86_CR0_PE))
7960 {
7961 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
7962 }
7963
7964 uint32_t u32EntryInfo;
7965 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7966 AssertRCBreak(rc);
7967 if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
7968 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7969 {
7970 HMVMX_CHECK_BREAK(u32Val & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
7971 }
7972
7973 /*
7974 * 64-bit checks.
7975 */
7976#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7977 if (HMVMX_IS_64BIT_HOST_MODE())
7978 {
7979 if ( fLongModeGuest
7980 && !fUnrestrictedGuest)
7981 {
7982 HMVMX_CHECK_BREAK(CPUMIsGuestPagingEnabledEx(pCtx), VMX_IGS_CR0_PG_LONGMODE);
7983 HMVMX_CHECK_BREAK((pCtx->cr4 & X86_CR4_PAE), VMX_IGS_CR4_PAE_LONGMODE);
7984 }
7985
7986 if ( !fLongModeGuest
7987 && (pCtx->cr4 & X86_CR4_PCIDE))
7988 {
7989 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
7990 }
7991
7992 /** @todo CR3 field must be such that bits 63:52 and bits in the range
7993 * 51:32 beyond the processor's physical-address width are 0. */
7994
7995 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7996 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
7997 {
7998 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
7999 }
8000
8001 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8002 AssertRCBreak(rc);
8003 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8004
8005 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8006 AssertRCBreak(rc);
8007 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8008 }
8009#endif
8010
8011 /*
8012 * PERF_GLOBAL MSR.
8013 */
8014 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8015 {
8016 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8017 AssertRCBreak(rc);
8018 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8019 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8020 }
8021
8022 /*
8023 * PAT MSR.
8024 */
8025 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8026 {
8027 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8028 AssertRCBreak(rc);
8029 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8030 for (unsigned i = 0; i < 8; i++)
8031 {
8032 uint8_t u8Val = (u64Val & 0x7);
8033 if ( u8Val != 0 /* UC */
8034 || u8Val != 1 /* WC */
8035 || u8Val != 4 /* WT */
8036 || u8Val != 5 /* WP */
8037 || u8Val != 6 /* WB */
8038 || u8Val != 7 /* UC- */)
8039 {
8040 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8041 }
8042 u64Val >>= 3;
8043 }
8044 }
8045
8046 /*
8047 * EFER MSR.
8048 */
8049 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8050 {
8051 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8052 AssertRCBreak(rc);
8053 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8054 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8055 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8056 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8057 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8058 || (u64Val & MSR_K6_EFER_LMA) == (pCtx->cr0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8059 }
8060
8061 /*
8062 * Segment registers.
8063 */
8064 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8065 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8066 if (!(u32Eflags & X86_EFL_VM))
8067 {
8068 /* CS */
8069 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8070 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8071 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8072 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8073 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8074 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8075 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8076 /* CS cannot be loaded with NULL in protected mode. */
8077 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8078 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8079 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8080 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8081 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8082 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8083 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8084 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8085 else
8086 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8087
8088 /* SS */
8089 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8090 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8091 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8092 if ( !(pCtx->cr0 & X86_CR0_PE)
8093 || pCtx->cs.Attr.n.u4Type == 3)
8094 {
8095 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8096 }
8097 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8098 {
8099 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8100 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8101 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8102 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8103 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8104 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8105 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8106 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8107 }
8108
8109 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8110 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8111 {
8112 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8113 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8114 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8115 || pCtx->ds.Attr.n.u4Type > 11
8116 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8117 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8118 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8119 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8120 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8121 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8122 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8123 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8124 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8125 }
8126 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8127 {
8128 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8129 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8130 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8131 || pCtx->es.Attr.n.u4Type > 11
8132 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8133 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8134 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8135 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8136 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8137 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8138 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8139 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8140 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8141 }
8142 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8143 {
8144 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8145 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8146 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8147 || pCtx->fs.Attr.n.u4Type > 11
8148 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8149 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8150 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8151 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8152 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8153 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8154 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8155 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8156 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8157 }
8158 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8159 {
8160 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8161 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8162 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8163 || pCtx->gs.Attr.n.u4Type > 11
8164 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8165 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8166 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8167 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8168 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8169 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8170 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8171 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8172 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8173 }
8174 /* 64-bit capable CPUs. */
8175#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8176 if (HMVMX_IS_64BIT_HOST_MODE())
8177 {
8178 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8179 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8180 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8181 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8182 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8183 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8184 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8185 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8186 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8187 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8188 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8189 }
8190#endif
8191 }
8192 else
8193 {
8194 /* V86 mode checks. */
8195 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8196 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8197 {
8198 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8199 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8200 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8201 }
8202 else
8203 {
8204 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8205 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8206 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8207 }
8208
8209 /* CS */
8210 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8211 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8212 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8213 /* SS */
8214 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8215 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8216 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8217 /* DS */
8218 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8219 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8220 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8221 /* ES */
8222 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8223 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8224 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8225 /* FS */
8226 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8227 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8228 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8229 /* GS */
8230 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8231 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8232 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8233 /* 64-bit capable CPUs. */
8234#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8235 if (HMVMX_IS_64BIT_HOST_MODE())
8236 {
8237 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8238 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8239 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8240 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8241 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8242 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8243 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8244 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8245 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8246 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8247 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8248 }
8249#endif
8250 }
8251
8252 /*
8253 * TR.
8254 */
8255 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8256 /* 64-bit capable CPUs. */
8257#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8258 if (HMVMX_IS_64BIT_HOST_MODE())
8259 {
8260 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8261 }
8262#endif
8263 if (fLongModeGuest)
8264 {
8265 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8266 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8267 }
8268 else
8269 {
8270 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8271 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8272 VMX_IGS_TR_ATTR_TYPE_INVALID);
8273 }
8274 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8275 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8276 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8277 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8278 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8279 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8280 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8281 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8282
8283 /*
8284 * GDTR and IDTR.
8285 */
8286#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8287 if (HMVMX_IS_64BIT_HOST_MODE())
8288 {
8289 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8290 AssertRCBreak(rc);
8291 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8292
8293 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8294 AssertRCBreak(rc);
8295 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8296 }
8297#endif
8298
8299 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8300 AssertRCBreak(rc);
8301 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8302
8303 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8304 AssertRCBreak(rc);
8305 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8306
8307 /*
8308 * Guest Non-Register State.
8309 */
8310 /* Activity State. */
8311 uint32_t u32ActivityState;
8312 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8313 AssertRCBreak(rc);
8314 HMVMX_CHECK_BREAK( !u32ActivityState
8315 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.msr.vmx_misc)),
8316 VMX_IGS_ACTIVITY_STATE_INVALID);
8317 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8318 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8319 uint32_t u32IntrState;
8320 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8321 AssertRCBreak(rc);
8322 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8323 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8324 {
8325 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8326 }
8327
8328 /** @todo Activity state and injecting interrupts. Left as a todo since we
8329 * currently don't use activity states but ACTIVE. */
8330
8331 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8332 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8333
8334 /* Guest interruptibility-state. */
8335 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8336 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8337 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8338 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8339 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8340 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8341 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8342 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8343 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8344 if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8345 {
8346 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8347 {
8348 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8349 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8350 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8351 }
8352 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8353 {
8354 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8355 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8356 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8357 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8358 }
8359 }
8360 /** @todo Assumes the processor is not in SMM. */
8361 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8362 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8363 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8364 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8365 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8366 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8367 && VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8368 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8369 {
8370 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8371 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8372 }
8373
8374 /* Pending debug exceptions. */
8375 if (HMVMX_IS_64BIT_HOST_MODE())
8376 {
8377 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8378 AssertRCBreak(rc);
8379 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8380 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8381 u32Val = u64Val; /* For pending debug exceptions checks below. */
8382 }
8383 else
8384 {
8385 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8386 AssertRCBreak(rc);
8387 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8388 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8389 }
8390
8391 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8392 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8393 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8394 {
8395 if ( (u32Eflags & X86_EFL_TF)
8396 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8397 {
8398 /* Bit 14 is PendingDebug.BS. */
8399 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8400 }
8401 if ( !(u32Eflags & X86_EFL_TF)
8402 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8403 {
8404 /* Bit 14 is PendingDebug.BS. */
8405 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8406 }
8407 }
8408
8409 /* VMCS link pointer. */
8410 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8411 AssertRCBreak(rc);
8412 if (u64Val != UINT64_C(0xffffffffffffffff))
8413 {
8414 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8415 /** @todo Bits beyond the processor's physical-address width MBZ. */
8416 /** @todo 32-bit located in memory referenced by value of this field (as a
8417 * physical address) must contain the processor's VMCS revision ID. */
8418 /** @todo SMM checks. */
8419 }
8420
8421 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8422
8423 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8424 if (uError == VMX_IGS_ERROR)
8425 uError = VMX_IGS_REASON_NOT_FOUND;
8426 } while (0);
8427
8428 pVCpu->hm.s.u32HMError = uError;
8429 return uError;
8430
8431#undef HMVMX_ERROR_BREAK
8432#undef HMVMX_CHECK_BREAK
8433#undef HMVMX_IS_CANONICAL
8434}
8435
8436/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8437/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8438/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8439
8440/** @name VM-exit handlers.
8441 * @{
8442 */
8443
8444/**
8445 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8446 */
8447HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8448{
8449 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8450 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8451 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8452#if HC_ARCH_BITS == 64 && defined(VBOX_WITH_VMMR0_DISABLE_PREEMPTION)
8453 Assert(ASMIntAreEnabled());
8454 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8455 return VINF_SUCCESS;
8456#endif
8457 return VINF_EM_RAW_INTERRUPT;
8458}
8459
8460
8461/**
8462 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8463 */
8464HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8465{
8466 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8467 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8468
8469 int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8470 AssertRCReturn(rc, rc);
8471
8472 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8473 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8474 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8475 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8476
8477 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8478 {
8479 /*
8480 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8481 * anything we inject is not going to cause a VM-exit directly for the event being injected.
8482 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8483 *
8484 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8485 */
8486 VMXDispatchHostNmi();
8487 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmi);
8488 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8489 return VINF_SUCCESS;
8490 }
8491
8492 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8493 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8494 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8495 {
8496 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8497 return VINF_SUCCESS;
8498 }
8499 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8500 {
8501 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8502 return rc;
8503 }
8504
8505 uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8506 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8507 switch (uIntrType)
8508 {
8509 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8510 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8511 /* no break */
8512 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8513 {
8514 switch (uVector)
8515 {
8516 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8517 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8518 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8519 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8520 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8521 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8522#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8523 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8524 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8525 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8526 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8527 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8528 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8529 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8530 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8531 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8532 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8533#endif
8534 default:
8535 {
8536 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8537 AssertRCReturn(rc, rc);
8538
8539 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8540 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8541 {
8542 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8543 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8544 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8545 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8546 AssertRCReturn(rc, rc);
8547 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8548 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8549 0 /* GCPtrFaultAddress */);
8550 AssertRCReturn(rc, rc);
8551 }
8552 else
8553 {
8554 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8555 pVCpu->hm.s.u32HMError = uVector;
8556 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8557 }
8558 break;
8559 }
8560 }
8561 break;
8562 }
8563
8564 default:
8565 {
8566 pVCpu->hm.s.u32HMError = uExitIntrInfo;
8567 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8568 AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8569 break;
8570 }
8571 }
8572 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8573 return rc;
8574}
8575
8576
8577/**
8578 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8579 */
8580HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8581{
8582 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8583
8584 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8585 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8586 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8587 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8588 AssertRCReturn(rc, rc);
8589
8590 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8591 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8592 return VINF_SUCCESS;
8593}
8594
8595
8596/**
8597 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8598 */
8599HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8600{
8601 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8602 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8603 pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8604 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8605}
8606
8607
8608/**
8609 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8610 */
8611HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8612{
8613 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8614 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8615 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8616}
8617
8618
8619/**
8620 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8621 */
8622HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8623{
8624 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8625 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8626 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8627}
8628
8629
8630/**
8631 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8632 */
8633HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8634{
8635 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8636 PVM pVM = pVCpu->CTX_SUFF(pVM);
8637 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8638 if (RT_LIKELY(rc == VINF_SUCCESS))
8639 {
8640 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8641 Assert(pVmxTransient->cbInstr == 2);
8642 }
8643 else
8644 {
8645 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8646 rc = VERR_EM_INTERPRETER;
8647 }
8648 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8649 return rc;
8650}
8651
8652
8653/**
8654 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8655 */
8656HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8657{
8658 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8659 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8660 AssertRCReturn(rc, rc);
8661
8662 if (pMixedCtx->cr4 & X86_CR4_SMXE)
8663 return VINF_EM_RAW_EMULATE_INSTR;
8664
8665 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8666 pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8667 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8668}
8669
8670
8671/**
8672 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8673 */
8674HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8675{
8676 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8677 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8678 AssertRCReturn(rc, rc);
8679
8680 PVM pVM = pVCpu->CTX_SUFF(pVM);
8681 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8682 if (RT_LIKELY(rc == VINF_SUCCESS))
8683 {
8684 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8685 Assert(pVmxTransient->cbInstr == 2);
8686 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8687 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8688 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8689 }
8690 else
8691 {
8692 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8693 rc = VERR_EM_INTERPRETER;
8694 }
8695 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8696 return rc;
8697}
8698
8699
8700/**
8701 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8702 */
8703HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8704{
8705 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8706 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8707 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8708 AssertRCReturn(rc, rc);
8709
8710 PVM pVM = pVCpu->CTX_SUFF(pVM);
8711 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8712 if (RT_LIKELY(rc == VINF_SUCCESS))
8713 {
8714 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8715 Assert(pVmxTransient->cbInstr == 3);
8716 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8717 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8718 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8719 }
8720 else
8721 {
8722 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8723 rc = VERR_EM_INTERPRETER;
8724 }
8725 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8726 return rc;
8727}
8728
8729
8730/**
8731 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8732 */
8733HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8734{
8735 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8736 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8737 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8738 AssertRCReturn(rc, rc);
8739
8740 PVM pVM = pVCpu->CTX_SUFF(pVM);
8741 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8742 if (RT_LIKELY(rc == VINF_SUCCESS))
8743 {
8744 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8745 Assert(pVmxTransient->cbInstr == 2);
8746 }
8747 else
8748 {
8749 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8750 rc = VERR_EM_INTERPRETER;
8751 }
8752 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8753 return rc;
8754}
8755
8756
8757/**
8758 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8759 */
8760HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8761{
8762 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8763 PVM pVM = pVCpu->CTX_SUFF(pVM);
8764 Assert(!pVM->hm.s.fNestedPaging);
8765
8766 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8767 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8768 AssertRCReturn(rc, rc);
8769
8770 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8771 rc = VBOXSTRICTRC_VAL(rc2);
8772 if (RT_LIKELY(rc == VINF_SUCCESS))
8773 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8774 else
8775 {
8776 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8777 pVmxTransient->uExitQualification, rc));
8778 }
8779 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8780 return rc;
8781}
8782
8783
8784/**
8785 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8786 */
8787HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8788{
8789 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8790 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8791 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8792 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8793 AssertRCReturn(rc, rc);
8794
8795 PVM pVM = pVCpu->CTX_SUFF(pVM);
8796 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8797 if (RT_LIKELY(rc == VINF_SUCCESS))
8798 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8799 else
8800 {
8801 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8802 rc = VERR_EM_INTERPRETER;
8803 }
8804 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8805 return rc;
8806}
8807
8808
8809/**
8810 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8811 */
8812HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8813{
8814 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8815 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8816 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8817 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8818 AssertRCReturn(rc, rc);
8819
8820 PVM pVM = pVCpu->CTX_SUFF(pVM);
8821 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8822 rc = VBOXSTRICTRC_VAL(rc2);
8823 if (RT_LIKELY( rc == VINF_SUCCESS
8824 || rc == VINF_EM_HALT))
8825 {
8826 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8827 AssertRCReturn(rc3, rc3);
8828
8829 if ( rc == VINF_EM_HALT
8830 && EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
8831 {
8832 rc = VINF_SUCCESS;
8833 }
8834 }
8835 else
8836 {
8837 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
8838 rc = VERR_EM_INTERPRETER;
8839 }
8840 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
8841 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
8842 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
8843 return rc;
8844}
8845
8846
8847/**
8848 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
8849 */
8850HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8851{
8852 /*
8853 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
8854 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
8855 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
8856 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
8857 */
8858 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8859 pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
8860 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8861}
8862
8863
8864/**
8865 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
8866 */
8867HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8868{
8869 /*
8870 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
8871 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
8872 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
8873 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
8874 */
8875 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8876 pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
8877 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8878}
8879
8880
8881/**
8882 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
8883 */
8884HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8885{
8886 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
8887 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8888 pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
8889 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8890}
8891
8892
8893/**
8894 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
8895 */
8896HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8897{
8898 /*
8899 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
8900 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
8901 * See Intel spec. 25.3 "Other Causes of VM-exits".
8902 */
8903 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8904 pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
8905 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8906}
8907
8908
8909/**
8910 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
8911 * VM-exit.
8912 */
8913HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8914{
8915 /*
8916 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
8917 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
8918 *
8919 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
8920 * See Intel spec. "23.8 Restrictions on VMX operation".
8921 */
8922 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8923 return VINF_SUCCESS;
8924}
8925
8926
8927/**
8928 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
8929 * VM-exit.
8930 */
8931HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8932{
8933 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8934 return VINF_EM_RESET;
8935}
8936
8937
8938/**
8939 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
8940 */
8941HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8942{
8943 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8944 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
8945 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8946 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8947 AssertRCReturn(rc, rc);
8948
8949 pMixedCtx->rip++;
8950 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
8951 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
8952 rc = VINF_SUCCESS;
8953 else
8954 rc = VINF_EM_HALT;
8955
8956 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8957 return rc;
8958}
8959
8960
8961/**
8962 * VM-exit handler for instructions that result in a #UD exception delivered to
8963 * the guest.
8964 */
8965HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8966{
8967 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8968 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
8969 return VINF_SUCCESS;
8970}
8971
8972
8973/**
8974 * VM-exit handler for expiry of the VMX preemption timer.
8975 */
8976HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8977{
8978 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8979
8980 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
8981 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8982
8983 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
8984 PVM pVM = pVCpu->CTX_SUFF(pVM);
8985 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
8986 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
8987 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
8988}
8989
8990
8991/**
8992 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
8993 */
8994HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8995{
8996 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8997
8998 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
8999 /** @todo check if XSETBV is supported by the recompiler. */
9000 return VERR_EM_INTERPRETER;
9001}
9002
9003
9004/**
9005 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9006 */
9007HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9008{
9009 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9010
9011 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9012 /** @todo implement EMInterpretInvpcid() */
9013 return VERR_EM_INTERPRETER;
9014}
9015
9016
9017/**
9018 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9019 * Error VM-exit.
9020 */
9021HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9022{
9023 uint32_t uIntrState;
9024 HMVMXHCUINTREG uHCReg;
9025 uint64_t u64Val;
9026 uint32_t u32Val;
9027
9028 int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
9029 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9030 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9031 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9032 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9033 AssertRCReturn(rc, rc);
9034
9035 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9036 NOREF(uInvalidReason);
9037
9038 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
9039 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9040 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9041 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9042
9043 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9044 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9045 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9046 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9047 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9048 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9049 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9050 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9051 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9052 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9053 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9054 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9055
9056 PVM pVM = pVCpu->CTX_SUFF(pVM);
9057 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9058
9059 return VERR_VMX_INVALID_GUEST_STATE;
9060}
9061
9062
9063/**
9064 * VM-exit handler for VM-entry failure due to an MSR-load
9065 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9066 */
9067HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9068{
9069 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9070 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9071}
9072
9073
9074/**
9075 * VM-exit handler for VM-entry failure due to a machine-check event
9076 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9077 */
9078HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9079{
9080 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9081 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9082}
9083
9084
9085/**
9086 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9087 * theory.
9088 */
9089HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9090{
9091 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9092 return VERR_VMX_UNDEFINED_EXIT_CODE;
9093}
9094
9095
9096/**
9097 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9098 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9099 * Conditional VM-exit.
9100 */
9101HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9102{
9103 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9104
9105 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9106 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9107 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9108 return VERR_EM_INTERPRETER;
9109 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9110 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9111}
9112
9113
9114/**
9115 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9116 */
9117HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9118{
9119 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9120
9121 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9122 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9123 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9124 return VERR_EM_INTERPRETER;
9125 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9126 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9127}
9128
9129
9130/**
9131 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9132 */
9133HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9134{
9135 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9136
9137 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9138 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9139 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9140 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9141 AssertRCReturn(rc, rc);
9142
9143 PVM pVM = pVCpu->CTX_SUFF(pVM);
9144 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9145 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9146 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9147 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9148
9149 if (RT_LIKELY(rc == VINF_SUCCESS))
9150 {
9151 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9152 Assert(pVmxTransient->cbInstr == 2);
9153 }
9154 return rc;
9155}
9156
9157
9158/**
9159 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9160 */
9161HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9162{
9163 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9164 PVM pVM = pVCpu->CTX_SUFF(pVM);
9165 int rc = VINF_SUCCESS;
9166
9167 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9168 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9169 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9170 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9171 AssertRCReturn(rc, rc);
9172 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9173
9174 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9175 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9176 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9177
9178 if (RT_LIKELY(rc == VINF_SUCCESS))
9179 {
9180 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9181
9182 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9183 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9184 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9185 {
9186 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9187 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9188 EMInterpretWrmsr() changes it. */
9189 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9190 }
9191 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9192 {
9193 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9194 AssertRCReturn(rc, rc);
9195 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
9196 }
9197 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9198 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9199
9200 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9201 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9202 {
9203 switch (pMixedCtx->ecx)
9204 {
9205 case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
9206 case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
9207 case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
9208 case MSR_K8_FS_BASE: /* no break */
9209 case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS; break;
9210 case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
9211 }
9212 }
9213#ifdef VBOX_STRICT
9214 else
9215 {
9216 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9217 switch (pMixedCtx->ecx)
9218 {
9219 case MSR_IA32_SYSENTER_CS:
9220 case MSR_IA32_SYSENTER_EIP:
9221 case MSR_IA32_SYSENTER_ESP:
9222 case MSR_K8_FS_BASE:
9223 case MSR_K8_GS_BASE:
9224 {
9225 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9226 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9227 }
9228
9229 case MSR_K8_LSTAR:
9230 case MSR_K6_STAR:
9231 case MSR_K8_SF_MASK:
9232 case MSR_K8_TSC_AUX:
9233 case MSR_K8_KERNEL_GS_BASE:
9234 {
9235 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9236 pMixedCtx->ecx));
9237 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9238 }
9239 }
9240 }
9241#endif /* VBOX_STRICT */
9242 }
9243 return rc;
9244}
9245
9246
9247/**
9248 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9249 */
9250HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9251{
9252 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9253
9254 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9255 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9256 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9257 return VERR_EM_INTERPRETER;
9258 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9259 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9260}
9261
9262
9263/**
9264 * VM-exit handler for when the TPR value is lowered below the specified
9265 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9266 */
9267HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9268{
9269 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9270 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9271
9272 /*
9273 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9274 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9275 * resume guest execution.
9276 */
9277 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9278 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9279 return VINF_SUCCESS;
9280}
9281
9282
9283/**
9284 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9285 * VM-exit.
9286 *
9287 * @retval VINF_SUCCESS when guest execution can continue.
9288 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9289 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9290 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9291 * recompiler.
9292 */
9293HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9294{
9295 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9296 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9297 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9298 AssertRCReturn(rc, rc);
9299
9300 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9301 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9302 PVM pVM = pVCpu->CTX_SUFF(pVM);
9303 switch (uAccessType)
9304 {
9305 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9306 {
9307#if 0
9308 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9309 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9310#else
9311 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9312 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9313 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9314#endif
9315 AssertRCReturn(rc, rc);
9316
9317 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9318 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9319 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9320 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9321
9322 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9323 {
9324 case 0: /* CR0 */
9325 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9326 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9327 break;
9328 case 2: /* C2 **/
9329 /* Nothing to do here, CR2 it's not part of the VMCS. */
9330 break;
9331 case 3: /* CR3 */
9332 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9333 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9334 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR3;
9335 break;
9336 case 4: /* CR4 */
9337 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9338 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR4;
9339 break;
9340 case 8: /* CR8 */
9341 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9342 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9343 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9344 break;
9345 default:
9346 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9347 break;
9348 }
9349
9350 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9351 break;
9352 }
9353
9354 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9355 {
9356 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9357 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9358 AssertRCReturn(rc, rc);
9359 Assert( !pVM->hm.s.fNestedPaging
9360 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9361 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9362
9363 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9364 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9365 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9366
9367 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9368 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9369 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9370 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9371 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9372 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9373 break;
9374 }
9375
9376 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9377 {
9378 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9379 AssertRCReturn(rc, rc);
9380 rc = EMInterpretCLTS(pVM, pVCpu);
9381 AssertRCReturn(rc, rc);
9382 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9383 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9384 Log4(("CRX CLTS write rc=%d\n", rc));
9385 break;
9386 }
9387
9388 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9389 {
9390 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9391 AssertRCReturn(rc, rc);
9392 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9393 if (RT_LIKELY(rc == VINF_SUCCESS))
9394 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9395 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9396 Log4(("CRX LMSW write rc=%d\n", rc));
9397 break;
9398 }
9399
9400 default:
9401 {
9402 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9403 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9404 }
9405 }
9406
9407 /* Validate possible error codes. */
9408 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9409 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9410 if (RT_SUCCESS(rc))
9411 {
9412 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9413 AssertRCReturn(rc2, rc2);
9414 }
9415
9416 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9417 return rc;
9418}
9419
9420
9421/**
9422 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9423 * VM-exit.
9424 */
9425HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9426{
9427 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9428 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9429
9430 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9431 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9432 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9433 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9434 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9435 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9436 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9437 AssertRCReturn(rc2, rc2);
9438
9439 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9440 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9441 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9442 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9443 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9444 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9445 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9446
9447 /* I/O operation lookup arrays. */
9448 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9449 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9450
9451 VBOXSTRICTRC rcStrict;
9452 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9453 const uint32_t cbInstr = pVmxTransient->cbInstr;
9454 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9455 PVM pVM = pVCpu->CTX_SUFF(pVM);
9456 if (fIOString)
9457 {
9458 /*
9459 * INS/OUTS - I/O String instruction.
9460 *
9461 * Use instruction-information if available, otherwise fall back on
9462 * interpreting the instruction.
9463 */
9464 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9465#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9466 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9467 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.msr.vmx_basic_info))
9468 {
9469 rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9470 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9471 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9472 AssertRCReturn(rc2, rc2);
9473 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9474 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9475 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9476 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9477 if (fIOWrite)
9478 {
9479 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9480 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9481 //if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9482 // hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9483 // pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9484 }
9485 else
9486 {
9487 AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9488 ("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9489 VERR_HMVMX_IPE_4);
9490 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9491 //if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9492 // hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9493 }
9494 }
9495 else
9496 {
9497 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9498 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9499 AssertRCReturn(rc2, rc2);
9500 rcStrict = IEMExecOne(pVCpu);
9501 }
9502 /** @todo IEM needs to be setting these flags somehow. */
9503 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9504 fUpdateRipAlready = true;
9505#else
9506 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9507 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9508 if (RT_SUCCESS(rcStrict))
9509 {
9510 if (fIOWrite)
9511 {
9512 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9513 (DISCPUMODE)pDis->uAddrMode, cbValue);
9514 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9515 }
9516 else
9517 {
9518 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9519 (DISCPUMODE)pDis->uAddrMode, cbValue);
9520 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9521 }
9522 }
9523 else
9524 {
9525 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9526 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9527 }
9528#endif
9529 }
9530 else
9531 {
9532 /*
9533 * IN/OUT - I/O instruction.
9534 */
9535 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9536 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9537 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9538 if (fIOWrite)
9539 {
9540 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9541 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9542 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9543 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9544 }
9545 else
9546 {
9547 uint32_t u32Result = 0;
9548 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9549 if (IOM_SUCCESS(rcStrict))
9550 {
9551 /* Save result of I/O IN instr. in AL/AX/EAX. */
9552 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
9553 }
9554 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9555 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9556 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9557 }
9558 }
9559
9560 if (IOM_SUCCESS(rcStrict))
9561 {
9562 if (!fUpdateRipAlready)
9563 {
9564 pMixedCtx->rip += cbInstr;
9565 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9566 }
9567
9568 /*
9569 * If any I/O breakpoints are armed, we need to check if one triggered
9570 * and take appropriate action.
9571 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9572 */
9573 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9574 AssertRCReturn(rc2, rc2);
9575
9576 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9577 * execution engines about whether hyper BPs and such are pending. */
9578 uint32_t const uDr7 = pMixedCtx->dr[7];
9579 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9580 && X86_DR7_ANY_RW_IO(uDr7)
9581 && (pMixedCtx->cr4 & X86_CR4_DE))
9582 || DBGFBpIsHwIoArmed(pVM)))
9583 {
9584 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9585 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
9586
9587 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9588 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9589 {
9590 /* Raise #DB. */
9591 if (fIsGuestDbgActive)
9592 ASMSetDR6(pMixedCtx->dr[6]);
9593 if (pMixedCtx->dr[7] != uDr7)
9594 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9595
9596 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9597 }
9598 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9599 else if ( rcStrict2 != VINF_SUCCESS
9600 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
9601 rcStrict = rcStrict2;
9602 }
9603 }
9604
9605#ifdef DEBUG
9606 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9607 Assert(!fIOWrite);
9608 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9609 Assert(fIOWrite);
9610 else
9611 {
9612 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9613 * statuses, that the VMM device and some others may return. See
9614 * IOM_SUCCESS() for guidance. */
9615 AssertMsg( RT_FAILURE(rcStrict)
9616 || rcStrict == VINF_SUCCESS
9617 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
9618 || rcStrict == VINF_EM_DBG_BREAKPOINT
9619 || rcStrict == VINF_EM_RAW_GUEST_TRAP
9620 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9621 }
9622#endif
9623
9624 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9625 return VBOXSTRICTRC_TODO(rcStrict);
9626}
9627
9628
9629/**
9630 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9631 * VM-exit.
9632 */
9633HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9634{
9635 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9636
9637 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
9638 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9639 AssertRCReturn(rc, rc);
9640 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9641 {
9642 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9643 AssertRCReturn(rc, rc);
9644 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9645 {
9646 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9647
9648 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9649 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9650 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9651 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9652 {
9653 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9654 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9655
9656 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9657 Assert(!pVCpu->hm.s.Event.fPending);
9658 pVCpu->hm.s.Event.fPending = true;
9659 pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9660 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9661 AssertRCReturn(rc, rc);
9662 if (fErrorCodeValid)
9663 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9664 else
9665 pVCpu->hm.s.Event.u32ErrCode = 0;
9666 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9667 && uVector == X86_XCPT_PF)
9668 {
9669 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9670 }
9671
9672 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9673 }
9674 }
9675 }
9676
9677 /** @todo Emulate task switch someday, currently just going back to ring-3 for
9678 * emulation. */
9679 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9680 return VERR_EM_INTERPRETER;
9681}
9682
9683
9684/**
9685 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9686 */
9687HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9688{
9689 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9690 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9691 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9692 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9693 AssertRCReturn(rc, rc);
9694 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9695 return VINF_EM_DBG_STEPPED;
9696}
9697
9698
9699/**
9700 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9701 */
9702HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9703{
9704 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9705
9706 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9707 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9708 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9709 return VINF_SUCCESS;
9710 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9711 return rc;
9712
9713#if 0
9714 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9715 * just sync the whole thing. */
9716 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9717#else
9718 /* Aggressive state sync. for now. */
9719 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9720 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9721 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9722#endif
9723 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9724 AssertRCReturn(rc, rc);
9725
9726 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9727 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9728 switch (uAccessType)
9729 {
9730 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9731 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9732 {
9733 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9734 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9735 {
9736 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9737 }
9738
9739 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9740 GCPhys &= PAGE_BASE_GC_MASK;
9741 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9742 PVM pVM = pVCpu->CTX_SUFF(pVM);
9743 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9744 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9745
9746 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9747 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9748 CPUMCTX2CORE(pMixedCtx), GCPhys);
9749 rc = VBOXSTRICTRC_VAL(rc2);
9750 Log4(("ApicAccess rc=%d\n", rc));
9751 if ( rc == VINF_SUCCESS
9752 || rc == VERR_PAGE_TABLE_NOT_PRESENT
9753 || rc == VERR_PAGE_NOT_PRESENT)
9754 {
9755 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
9756 | HM_CHANGED_VMX_GUEST_APIC_STATE;
9757 rc = VINF_SUCCESS;
9758 }
9759 break;
9760 }
9761
9762 default:
9763 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9764 rc = VINF_EM_RAW_EMULATE_INSTR;
9765 break;
9766 }
9767
9768 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9769 return rc;
9770}
9771
9772
9773/**
9774 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9775 * VM-exit.
9776 */
9777HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9778{
9779 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9780
9781 /* We should -not- get this VM-exit if the guest is debugging. */
9782 if (CPUMIsGuestDebugStateActive(pVCpu))
9783 {
9784 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9785 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9786 }
9787
9788 int rc = VERR_INTERNAL_ERROR_5;
9789 if ( !DBGFIsStepping(pVCpu)
9790 && !pVCpu->hm.s.fSingleInstruction
9791 && !CPUMIsHyperDebugStateActive(pVCpu))
9792 {
9793 /* Don't intercept MOV DRx and #DB any more. */
9794 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
9795 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9796 AssertRCReturn(rc, rc);
9797
9798 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9799 {
9800#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9801 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
9802 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9803 AssertRCReturn(rc, rc);
9804#endif
9805 }
9806
9807 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9808 PVM pVM = pVCpu->CTX_SUFF(pVM);
9809 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9810 Assert(CPUMIsGuestDebugStateActive(pVCpu));
9811
9812#ifdef VBOX_WITH_STATISTICS
9813 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9814 AssertRCReturn(rc, rc);
9815 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9816 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9817 else
9818 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9819#endif
9820 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
9821 return VINF_SUCCESS;
9822 }
9823
9824 /*
9825 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
9826 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
9827 */
9828 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9829 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9830 AssertRCReturn(rc, rc);
9831
9832 PVM pVM = pVCpu->CTX_SUFF(pVM);
9833 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9834 {
9835 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9836 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
9837 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
9838 if (RT_SUCCESS(rc))
9839 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9840 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9841 }
9842 else
9843 {
9844 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9845 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
9846 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
9847 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9848 }
9849
9850 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9851 if (RT_SUCCESS(rc))
9852 {
9853 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9854 AssertRCReturn(rc2, rc2);
9855 }
9856 return rc;
9857}
9858
9859
9860/**
9861 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
9862 * Conditional VM-exit.
9863 */
9864HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9865{
9866 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9867 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9868
9869 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9870 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9871 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9872 return VINF_SUCCESS;
9873 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9874 return rc;
9875
9876 RTGCPHYS GCPhys = 0;
9877 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9878
9879#if 0
9880 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9881#else
9882 /* Aggressive state sync. for now. */
9883 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9884 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9885 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9886#endif
9887 AssertRCReturn(rc, rc);
9888
9889 /*
9890 * If we succeed, resume guest execution.
9891 * If we fail in interpreting the instruction because we couldn't get the guest physical address
9892 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
9893 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
9894 * weird case. See @bugref{6043}.
9895 */
9896 PVM pVM = pVCpu->CTX_SUFF(pVM);
9897 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
9898 rc = VBOXSTRICTRC_VAL(rc2);
9899 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
9900 if ( rc == VINF_SUCCESS
9901 || rc == VERR_PAGE_TABLE_NOT_PRESENT
9902 || rc == VERR_PAGE_NOT_PRESENT)
9903 {
9904 /* Successfully handled MMIO operation. */
9905 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
9906 | HM_CHANGED_VMX_GUEST_APIC_STATE;
9907 rc = VINF_SUCCESS;
9908 }
9909 return rc;
9910}
9911
9912
9913/**
9914 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
9915 * VM-exit.
9916 */
9917HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9918{
9919 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9920 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9921
9922 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9923 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9924 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9925 return VINF_SUCCESS;
9926 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9927 return rc;
9928
9929 RTGCPHYS GCPhys = 0;
9930 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9931 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9932#if 0
9933 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9934#else
9935 /* Aggressive state sync. for now. */
9936 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9937 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9938 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9939#endif
9940 AssertRCReturn(rc, rc);
9941
9942 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
9943 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
9944
9945 RTGCUINT uErrorCode = 0;
9946 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
9947 uErrorCode |= X86_TRAP_PF_ID;
9948 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
9949 uErrorCode |= X86_TRAP_PF_RW;
9950 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
9951 uErrorCode |= X86_TRAP_PF_P;
9952
9953 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
9954
9955 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
9956 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
9957
9958 /* Handle the pagefault trap for the nested shadow table. */
9959 PVM pVM = pVCpu->CTX_SUFF(pVM);
9960 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
9961 TRPMResetTrap(pVCpu);
9962
9963 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
9964 if ( rc == VINF_SUCCESS
9965 || rc == VERR_PAGE_TABLE_NOT_PRESENT
9966 || rc == VERR_PAGE_NOT_PRESENT)
9967 {
9968 /* Successfully synced our nested page tables. */
9969 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
9970 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
9971 return VINF_SUCCESS;
9972 }
9973
9974 Log4(("EPT return to ring-3 rc=%d\n"));
9975 return rc;
9976}
9977
9978/** @} */
9979
9980/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9981/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
9982/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9983
9984/** @name VM-exit exception handlers.
9985 * @{
9986 */
9987
9988/**
9989 * VM-exit exception handler for #MF (Math Fault: floating point exception).
9990 */
9991static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9992{
9993 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
9994 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
9995
9996 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9997 AssertRCReturn(rc, rc);
9998
9999 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10000 {
10001 /* Old-style FPU error reporting needs some extra work. */
10002 /** @todo don't fall back to the recompiler, but do it manually. */
10003 return VERR_EM_INTERPRETER;
10004 }
10005
10006 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10007 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10008 return rc;
10009}
10010
10011
10012/**
10013 * VM-exit exception handler for #BP (Breakpoint exception).
10014 */
10015static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10016{
10017 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10018 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10019
10020 /** @todo Try optimize this by not saving the entire guest state unless
10021 * really needed. */
10022 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10023 AssertRCReturn(rc, rc);
10024
10025 PVM pVM = pVCpu->CTX_SUFF(pVM);
10026 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10027 if (rc == VINF_EM_RAW_GUEST_TRAP)
10028 {
10029 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10030 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10031 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10032 AssertRCReturn(rc, rc);
10033
10034 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10035 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10036 }
10037
10038 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10039 return rc;
10040}
10041
10042
10043/**
10044 * VM-exit exception handler for #DB (Debug exception).
10045 */
10046static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10047{
10048 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10049 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10050 Log6(("XcptDB\n"));
10051
10052 /*
10053 * Get the DR6-like values from the exit qualification and pass it to DBGF
10054 * for processing.
10055 */
10056 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10057 AssertRCReturn(rc, rc);
10058
10059 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10060 uint64_t uDR6 = X86_DR6_INIT_VAL;
10061 uDR6 |= ( pVmxTransient->uExitQualification
10062 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10063
10064 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10065 if (rc == VINF_EM_RAW_GUEST_TRAP)
10066 {
10067 /*
10068 * The exception was for the guest. Update DR6, DR7.GD and
10069 * IA32_DEBUGCTL.LBR before forwarding it.
10070 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10071 */
10072 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10073 pMixedCtx->dr[6] |= uDR6;
10074 if (CPUMIsGuestDebugStateActive(pVCpu))
10075 ASMSetDR6(pMixedCtx->dr[6]);
10076
10077 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10078 AssertRCReturn(rc, rc);
10079
10080 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10081 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10082
10083 /* Paranoia. */
10084 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10085 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10086
10087 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10088 AssertRCReturn(rc, rc);
10089
10090 /*
10091 * Raise #DB in the guest.
10092 */
10093 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10094 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10095 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10096 AssertRCReturn(rc, rc);
10097 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10098 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10099 return VINF_SUCCESS;
10100 }
10101
10102 /*
10103 * Not a guest trap, must be a hypervisor related debug event then.
10104 * Update DR6 in case someone is interested in it.
10105 */
10106 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10107 AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
10108 CPUMSetHyperDR6(pVCpu, uDR6);
10109
10110 return rc;
10111}
10112
10113
10114/**
10115 * VM-exit exception handler for #NM (Device-not-available exception: floating
10116 * point exception).
10117 */
10118static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10119{
10120 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10121
10122#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10123 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
10124#endif
10125
10126 /* We require CR0 and EFER. EFER is always up-to-date. */
10127 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10128 AssertRCReturn(rc, rc);
10129
10130 /* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10131 PVM pVM = pVCpu->CTX_SUFF(pVM);
10132 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
10133 if (rc == VINF_SUCCESS)
10134 {
10135 Assert(CPUMIsGuestFPUStateActive(pVCpu));
10136 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
10137 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10138 return VINF_SUCCESS;
10139 }
10140
10141 /* Forward #NM to the guest. */
10142 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10143 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10144 AssertRCReturn(rc, rc);
10145 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10146 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10147 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10148 return rc;
10149}
10150
10151
10152/**
10153 * VM-exit exception handler for #GP (General-protection exception).
10154 *
10155 * @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
10156 */
10157static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10158{
10159 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10160 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10161
10162 int rc = VERR_INTERNAL_ERROR_5;
10163 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10164 {
10165#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10166 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10167 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10168 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10169 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10170 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10171 AssertRCReturn(rc, rc);
10172 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10173 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10174 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10175 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10176 return rc;
10177#else
10178 /* We don't intercept #GP. */
10179 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10180 return VERR_VMX_UNEXPECTED_EXCEPTION;
10181#endif
10182 }
10183
10184 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10185 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10186
10187 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10188 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10189 AssertRCReturn(rc, rc);
10190
10191 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10192 uint32_t cbOp = 0;
10193 PVM pVM = pVCpu->CTX_SUFF(pVM);
10194 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10195 if (RT_SUCCESS(rc))
10196 {
10197 rc = VINF_SUCCESS;
10198 Assert(cbOp == pDis->cbInstr);
10199 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10200 switch (pDis->pCurInstr->uOpcode)
10201 {
10202 case OP_CLI:
10203 {
10204 pMixedCtx->eflags.Bits.u1IF = 0;
10205 pMixedCtx->rip += pDis->cbInstr;
10206 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10207 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10208 break;
10209 }
10210
10211 case OP_STI:
10212 {
10213 pMixedCtx->eflags.Bits.u1IF = 1;
10214 pMixedCtx->rip += pDis->cbInstr;
10215 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10216 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10217 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10218 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10219 break;
10220 }
10221
10222 case OP_HLT:
10223 {
10224 rc = VINF_EM_HALT;
10225 pMixedCtx->rip += pDis->cbInstr;
10226 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
10227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10228 break;
10229 }
10230
10231 case OP_POPF:
10232 {
10233 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10234 uint32_t cbParm = 0;
10235 uint32_t uMask = 0;
10236 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10237 {
10238 cbParm = 4;
10239 uMask = 0xffffffff;
10240 }
10241 else
10242 {
10243 cbParm = 2;
10244 uMask = 0xffff;
10245 }
10246
10247 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10248 RTGCPTR GCPtrStack = 0;
10249 X86EFLAGS Eflags;
10250 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10251 &GCPtrStack);
10252 if (RT_SUCCESS(rc))
10253 {
10254 Assert(sizeof(Eflags.u32) >= cbParm);
10255 Eflags.u32 = 0;
10256 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10257 }
10258 if (RT_FAILURE(rc))
10259 {
10260 rc = VERR_EM_INTERPRETER;
10261 break;
10262 }
10263 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10264 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10265 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10266 /* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10267 pMixedCtx->eflags.Bits.u1RF = 0;
10268 pMixedCtx->esp += cbParm;
10269 pMixedCtx->esp &= uMask;
10270 pMixedCtx->rip += pDis->cbInstr;
10271 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10272 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10273 break;
10274 }
10275
10276 case OP_PUSHF:
10277 {
10278 uint32_t cbParm = 0;
10279 uint32_t uMask = 0;
10280 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10281 {
10282 cbParm = 4;
10283 uMask = 0xffffffff;
10284 }
10285 else
10286 {
10287 cbParm = 2;
10288 uMask = 0xffff;
10289 }
10290
10291 /* Get the stack pointer & push the contents of eflags onto the stack. */
10292 RTGCPTR GCPtrStack = 0;
10293 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10294 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10295 if (RT_FAILURE(rc))
10296 {
10297 rc = VERR_EM_INTERPRETER;
10298 break;
10299 }
10300 X86EFLAGS Eflags = pMixedCtx->eflags;
10301 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10302 Eflags.Bits.u1RF = 0;
10303 Eflags.Bits.u1VM = 0;
10304
10305 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10306 if (RT_FAILURE(rc))
10307 {
10308 rc = VERR_EM_INTERPRETER;
10309 break;
10310 }
10311 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10312 pMixedCtx->esp -= cbParm;
10313 pMixedCtx->esp &= uMask;
10314 pMixedCtx->rip += pDis->cbInstr;
10315 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP;
10316 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10317 break;
10318 }
10319
10320 case OP_IRET:
10321 {
10322 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10323 * instruction reference. */
10324 RTGCPTR GCPtrStack = 0;
10325 uint32_t uMask = 0xffff;
10326 uint16_t aIretFrame[3];
10327 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10328 {
10329 rc = VERR_EM_INTERPRETER;
10330 break;
10331 }
10332 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10333 &GCPtrStack);
10334 if (RT_SUCCESS(rc))
10335 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10336 if (RT_FAILURE(rc))
10337 {
10338 rc = VERR_EM_INTERPRETER;
10339 break;
10340 }
10341 pMixedCtx->eip = 0;
10342 pMixedCtx->ip = aIretFrame[0];
10343 pMixedCtx->cs.Sel = aIretFrame[1];
10344 pMixedCtx->cs.ValidSel = aIretFrame[1];
10345 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10346 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10347 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10348 pMixedCtx->sp += sizeof(aIretFrame);
10349 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_SEGMENT_REGS | HM_CHANGED_GUEST_RSP
10350 | HM_CHANGED_GUEST_RFLAGS;
10351 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10352 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10353 break;
10354 }
10355
10356 case OP_INT:
10357 {
10358 uint16_t uVector = pDis->Param1.uValue & 0xff;
10359 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10360 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10361 break;
10362 }
10363
10364 case OP_INTO:
10365 {
10366 if (pMixedCtx->eflags.Bits.u1OF)
10367 {
10368 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10369 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10370 }
10371 break;
10372 }
10373
10374 default:
10375 {
10376 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10377 EMCODETYPE_SUPERVISOR);
10378 rc = VBOXSTRICTRC_VAL(rc2);
10379 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
10380 Log4(("#GP rc=%Rrc\n", rc));
10381 break;
10382 }
10383 }
10384 }
10385 else
10386 rc = VERR_EM_INTERPRETER;
10387
10388 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10389 ("#GP Unexpected rc=%Rrc\n", rc));
10390 return rc;
10391}
10392
10393
10394/**
10395 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10396 * the exception reported in the VMX transient structure back into the VM.
10397 *
10398 * @remarks Requires uExitIntrInfo in the VMX transient structure to be
10399 * up-to-date.
10400 */
10401static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10402{
10403 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10404
10405 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10406 hmR0VmxCheckExitDueToEventDelivery(). */
10407 int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10408 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10409 AssertRCReturn(rc, rc);
10410 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10411
10412 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10413 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10414 return VINF_SUCCESS;
10415}
10416
10417
10418/**
10419 * VM-exit exception handler for #PF (Page-fault exception).
10420 */
10421static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10422{
10423 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10424 PVM pVM = pVCpu->CTX_SUFF(pVM);
10425 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10426 rc |= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10427 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10428 AssertRCReturn(rc, rc);
10429
10430#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
10431 if (pVM->hm.s.fNestedPaging)
10432 {
10433 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10434 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10435 {
10436 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10437 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10438 0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10439 }
10440 else
10441 {
10442 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10443 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10444 Log4(("Pending #DF due to vectoring #PF. NP\n"));
10445 }
10446 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10447 return rc;
10448 }
10449#else
10450 Assert(!pVM->hm.s.fNestedPaging);
10451#endif
10452
10453 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10454 AssertRCReturn(rc, rc);
10455
10456 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10457 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10458
10459 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10460 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10461 (RTGCPTR)pVmxTransient->uExitQualification);
10462
10463 Log4(("#PF: rc=%Rrc\n", rc));
10464 if (rc == VINF_SUCCESS)
10465 {
10466 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10467 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
10468 * memory? We don't update the whole state here... */
10469 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10470 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10471 TRPMResetTrap(pVCpu);
10472 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10473 return rc;
10474 }
10475 else if (rc == VINF_EM_RAW_GUEST_TRAP)
10476 {
10477 if (!pVmxTransient->fVectoringPF)
10478 {
10479 /* It's a guest page fault and needs to be reflected to the guest. */
10480 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10481 TRPMResetTrap(pVCpu);
10482 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10483 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10484 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10485 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10486 }
10487 else
10488 {
10489 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10490 TRPMResetTrap(pVCpu);
10491 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10492 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10493 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10494 }
10495
10496 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10497 return VINF_SUCCESS;
10498 }
10499
10500 TRPMResetTrap(pVCpu);
10501 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10502 return rc;
10503}
10504
10505/** @} */
10506
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette