VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 48395

Last change on this file since 48395 was 48394, checked in by vboxsync, 11 years ago

VMM/HMVMXR0: Address possible deadlock situation with the PGM lock. Reenabled thread-context hooks again.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 447.0 KB
Line 
1/* $Id: HMVMXR0.cpp 48394 2013-09-09 15:54:51Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#endif
45
46
47/*******************************************************************************
48* Defined Constants And Macros *
49*******************************************************************************/
50#if defined(RT_ARCH_AMD64)
51# define HMVMX_IS_64BIT_HOST_MODE() (true)
52typedef RTHCUINTREG HMVMXHCUINTREG;
53#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54extern "C" uint32_t g_fVMXIs64bitHost;
55# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56typedef uint64_t HMVMXHCUINTREG;
57#else
58# define HMVMX_IS_64BIT_HOST_MODE() (false)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#endif
61
62/** Use the function table. */
63#define HMVMX_USE_FUNCTION_TABLE
64
65/** Determine which tagged-TLB flush handler to use. */
66#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71/** @name Updated-guest-state flags.
72 * @{ */
73#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94 | HMVMX_UPDATED_GUEST_RSP \
95 | HMVMX_UPDATED_GUEST_RFLAGS \
96 | HMVMX_UPDATED_GUEST_CR0 \
97 | HMVMX_UPDATED_GUEST_CR3 \
98 | HMVMX_UPDATED_GUEST_CR4 \
99 | HMVMX_UPDATED_GUEST_GDTR \
100 | HMVMX_UPDATED_GUEST_IDTR \
101 | HMVMX_UPDATED_GUEST_LDTR \
102 | HMVMX_UPDATED_GUEST_TR \
103 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104 | HMVMX_UPDATED_GUEST_DEBUG \
105 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112 | HMVMX_UPDATED_GUEST_APIC_STATE)
113/** @} */
114
115/** @name
116 * Flags to skip redundant reads of some common VMCS fields that are not part of
117 * the guest-CPU state but are in the transient structure.
118 */
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually (except #PF). #NM is also
143 * handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
144 * intercepted even in real-mode if we have Nested Paging support.
145 */
146#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
147 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
148 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
149 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
150 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
151 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
152 | RT_BIT(X86_XCPT_XF))
153
154/**
155 * Exception bitmap mask for all contributory exceptions.
156 *
157 * Page fault is deliberately excluded here as it's conditional as to whether
158 * it's contributory or benign. Page faults are handled separately.
159 */
160#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
161 | RT_BIT(X86_XCPT_DE))
162
163/** Maximum VM-instruction error number. */
164#define HMVMX_INSTR_ERROR_MAX 28
165
166/** Profiling macro. */
167#ifdef HM_PROFILE_EXIT_DISPATCH
168# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
169# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
170#else
171# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
173#endif
174
175/** Assert that preemption is disabled or covered by thread-context hooks. */
176#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
177 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
178
179/** Assert that we haven't migrated CPUs when thread-context hooks are not
180 * used. */
181#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
182 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
183 ("Illegal migration! Entered on CPU %u Current %u\n", \
184 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
185
186/*******************************************************************************
187* Structures and Typedefs *
188*******************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG uEflags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
203 uint64_t u64LStarMsr;
204 /** The guest's TPR value used for TPR shadowing. */
205 uint8_t u8GuestTpr;
206 /** Alignment. */
207 uint8_t abAlignment0[7];
208
209 /** The basic VM-exit reason. */
210 uint16_t uExitReason;
211 /** Alignment. */
212 uint16_t u16Alignment0;
213 /** The VM-exit interruption error code. */
214 uint32_t uExitIntrErrorCode;
215 /** The VM-exit exit qualification. */
216 uint64_t uExitQualification;
217
218 /** The VM-exit interruption-information field. */
219 uint32_t uExitIntrInfo;
220 /** The VM-exit instruction-length field. */
221 uint32_t cbInstr;
222 /** The VM-exit instruction-information field. */
223 union
224 {
225 /** Plain unsigned int representation. */
226 uint32_t u;
227 /** INS and OUTS information. */
228 struct
229 {
230 uint32_t u6Reserved0 : 6;
231 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
232 uint32_t u3AddrSize : 3;
233 uint32_t u5Reserved1 : 5;
234 /** The segment register (X86_SREG_XXX). */
235 uint32_t iSegReg : 3;
236 uint32_t uReserved2 : 14;
237 } StrIo;
238 } ExitInstrInfo;
239 /** Whether the VM-entry failed or not. */
240 bool fVMEntryFailed;
241 /** Alignment. */
242 uint8_t abAlignment1[3];
243
244 /** The VM-entry interruption-information field. */
245 uint32_t uEntryIntrInfo;
246 /** The VM-entry exception error code field. */
247 uint32_t uEntryXcptErrorCode;
248 /** The VM-entry instruction length field. */
249 uint32_t cbEntryInstr;
250
251 /** IDT-vectoring information field. */
252 uint32_t uIdtVectoringInfo;
253 /** IDT-vectoring error code. */
254 uint32_t uIdtVectoringErrorCode;
255
256 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
257 uint32_t fVmcsFieldsRead;
258 /** Whether TSC-offsetting should be setup before VM-entry. */
259 bool fUpdateTscOffsettingAndPreemptTimer;
260 /** Whether the VM-exit was caused by a page-fault during delivery of a
261 * contributory exception or a page-fault. */
262 bool fVectoringPF;
263} VMXTRANSIENT;
264AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
265AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
266AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
267AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
268/** Pointer to VMX transient state. */
269typedef VMXTRANSIENT *PVMXTRANSIENT;
270
271
272/**
273 * MSR-bitmap read permissions.
274 */
275typedef enum VMXMSREXITREAD
276{
277 /** Reading this MSR causes a VM-exit. */
278 VMXMSREXIT_INTERCEPT_READ = 0xb,
279 /** Reading this MSR does not cause a VM-exit. */
280 VMXMSREXIT_PASSTHRU_READ
281} VMXMSREXITREAD;
282
283/**
284 * MSR-bitmap write permissions.
285 */
286typedef enum VMXMSREXITWRITE
287{
288 /** Writing to this MSR causes a VM-exit. */
289 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
290 /** Writing to this MSR does not cause a VM-exit. */
291 VMXMSREXIT_PASSTHRU_WRITE
292} VMXMSREXITWRITE;
293
294/**
295 * VMX VM-exit handler.
296 *
297 * @returns VBox status code.
298 * @param pVCpu Pointer to the VMCPU.
299 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
300 * out-of-sync. Make sure to update the required
301 * fields before using them.
302 * @param pVmxTransient Pointer to the VMX-transient structure.
303 */
304#ifndef HMVMX_USE_FUNCTION_TABLE
305typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306#else
307typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308/** Pointer to VM-exit handler. */
309typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
310#endif
311
312
313/*******************************************************************************
314* Internal Functions *
315*******************************************************************************/
316static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
317static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
318static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
319static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
320 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
321#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
322static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
323#endif
324#ifndef HMVMX_USE_FUNCTION_TABLE
325DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
326# define HMVMX_EXIT_DECL static int
327#else
328# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
329#endif
330
331/** @name VM-exit handlers.
332 * @{
333 */
334static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
335static FNVMXEXITHANDLER hmR0VmxExitExtInt;
336static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
337static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
338static FNVMXEXITHANDLER hmR0VmxExitSipi;
339static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
340static FNVMXEXITHANDLER hmR0VmxExitSmi;
341static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
342static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
343static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
344static FNVMXEXITHANDLER hmR0VmxExitCpuid;
345static FNVMXEXITHANDLER hmR0VmxExitGetsec;
346static FNVMXEXITHANDLER hmR0VmxExitHlt;
347static FNVMXEXITHANDLER hmR0VmxExitInvd;
348static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
349static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
350static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
351static FNVMXEXITHANDLER hmR0VmxExitRsm;
352static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
353static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
354static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
355static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
356static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
357static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
358static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
359static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
360static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
361static FNVMXEXITHANDLER hmR0VmxExitMwait;
362static FNVMXEXITHANDLER hmR0VmxExitMtf;
363static FNVMXEXITHANDLER hmR0VmxExitMonitor;
364static FNVMXEXITHANDLER hmR0VmxExitPause;
365static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
366static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
367static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
368static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
369static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
370static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
371static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
372static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
373static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
374static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
375static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
376static FNVMXEXITHANDLER hmR0VmxExitRdrand;
377static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
378/** @} */
379
380static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
381static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
382static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
383static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
384static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
385static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
386static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
387static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
388
389/*******************************************************************************
390* Global Variables *
391*******************************************************************************/
392#ifdef HMVMX_USE_FUNCTION_TABLE
393
394/**
395 * VMX_EXIT dispatch table.
396 */
397static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
398{
399 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
400 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
401 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
402 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
403 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
404 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
405 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
406 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
407 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
408 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
409 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
410 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
411 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
412 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
413 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
414 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
415 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
416 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
417 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
418 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
419 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
420 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
421 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
422 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
423 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
424 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
425 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
426 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
427 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
428 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
429 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
430 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
431 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
432 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
433 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
434 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
435 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
436 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
437 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
438 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
439 /* 40 UNDEFINED */ hmR0VmxExitPause,
440 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
441 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
442 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
443 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
444 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
445 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
446 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
447 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
448 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
449 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
450 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
451 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
452 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
453 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
454 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
455 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
456 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
457 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
458 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
459};
460#endif /* HMVMX_USE_FUNCTION_TABLE */
461
462#ifdef VBOX_STRICT
463static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
464{
465 /* 0 */ "(Not Used)",
466 /* 1 */ "VMCALL executed in VMX root operation.",
467 /* 2 */ "VMCLEAR with invalid physical address.",
468 /* 3 */ "VMCLEAR with VMXON pointer.",
469 /* 4 */ "VMLAUNCH with non-clear VMCS.",
470 /* 5 */ "VMRESUME with non-launched VMCS.",
471 /* 6 */ "VMRESUME after VMXOFF",
472 /* 7 */ "VM entry with invalid control fields.",
473 /* 8 */ "VM entry with invalid host state fields.",
474 /* 9 */ "VMPTRLD with invalid physical address.",
475 /* 10 */ "VMPTRLD with VMXON pointer.",
476 /* 11 */ "VMPTRLD with incorrect revision identifier.",
477 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
478 /* 13 */ "VMWRITE to read-only VMCS component.",
479 /* 14 */ "(Not Used)",
480 /* 15 */ "VMXON executed in VMX root operation.",
481 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
482 /* 17 */ "VM entry with non-launched executing VMCS.",
483 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
484 /* 19 */ "VMCALL with non-clear VMCS.",
485 /* 20 */ "VMCALL with invalid VM-exit control fields.",
486 /* 21 */ "(Not Used)",
487 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
488 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
489 /* 24 */ "VMCALL with invalid SMM-monitor features.",
490 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
491 /* 26 */ "VM entry with events blocked by MOV SS.",
492 /* 27 */ "(Not Used)",
493 /* 28 */ "Invalid operand to INVEPT/INVVPID."
494};
495#endif /* VBOX_STRICT */
496
497
498
499/**
500 * Updates the VM's last error record. If there was a VMX instruction error,
501 * reads the error data from the VMCS and updates VCPU's last error record as
502 * well.
503 *
504 * @param pVM Pointer to the VM.
505 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
506 * VERR_VMX_UNABLE_TO_START_VM or
507 * VERR_VMX_INVALID_VMCS_FIELD).
508 * @param rc The error code.
509 */
510static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
511{
512 AssertPtr(pVM);
513 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
514 || rc == VERR_VMX_UNABLE_TO_START_VM)
515 {
516 AssertPtrReturnVoid(pVCpu);
517 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
518 }
519 pVM->hm.s.lLastError = rc;
520}
521
522
523/**
524 * Reads the VM-entry interruption-information field from the VMCS into the VMX
525 * transient structure.
526 *
527 * @returns VBox status code.
528 * @param pVmxTransient Pointer to the VMX transient structure.
529 *
530 * @remarks No-long-jump zone!!!
531 */
532DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
533{
534 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
535 AssertRCReturn(rc, rc);
536 return VINF_SUCCESS;
537}
538
539
540/**
541 * Reads the VM-entry exception error code field from the VMCS into
542 * the VMX transient structure.
543 *
544 * @returns VBox status code.
545 * @param pVmxTransient Pointer to the VMX transient structure.
546 *
547 * @remarks No-long-jump zone!!!
548 */
549DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
550{
551 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
552 AssertRCReturn(rc, rc);
553 return VINF_SUCCESS;
554}
555
556
557/**
558 * Reads the VM-entry exception error code field from the VMCS into
559 * the VMX transient structure.
560 *
561 * @returns VBox status code.
562 * @param pVCpu Pointer to the VMCPU.
563 * @param pVmxTransient Pointer to the VMX transient structure.
564 *
565 * @remarks No-long-jump zone!!!
566 */
567DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
568{
569 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
570 AssertRCReturn(rc, rc);
571 return VINF_SUCCESS;
572}
573
574
575/**
576 * Reads the VM-exit interruption-information field from the VMCS into the VMX
577 * transient structure.
578 *
579 * @returns VBox status code.
580 * @param pVCpu Pointer to the VMCPU.
581 * @param pVmxTransient Pointer to the VMX transient structure.
582 */
583DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
584{
585 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
586 {
587 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
588 AssertRCReturn(rc, rc);
589 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
590 }
591 return VINF_SUCCESS;
592}
593
594
595/**
596 * Reads the VM-exit interruption error code from the VMCS into the VMX
597 * transient structure.
598 *
599 * @returns VBox status code.
600 * @param pVCpu Pointer to the VMCPU.
601 * @param pVmxTransient Pointer to the VMX transient structure.
602 */
603DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
604{
605 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
606 {
607 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
608 AssertRCReturn(rc, rc);
609 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
610 }
611 return VINF_SUCCESS;
612}
613
614
615/**
616 * Reads the VM-exit instruction length field from the VMCS into the VMX
617 * transient structure.
618 *
619 * @returns VBox status code.
620 * @param pVCpu Pointer to the VMCPU.
621 * @param pVmxTransient Pointer to the VMX transient structure.
622 */
623DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
624{
625 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
626 {
627 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
628 AssertRCReturn(rc, rc);
629 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
630 }
631 return VINF_SUCCESS;
632}
633
634
635/**
636 * Reads the VM-exit instruction-information field from the VMCS into
637 * the VMX transient structure.
638 *
639 * @returns VBox status code.
640 * @param pVCpu The cross context per CPU structure.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
648 AssertRCReturn(rc, rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the exit qualification from the VMCS into the VMX transient structure.
657 *
658 * @returns VBox status code.
659 * @param pVCpu Pointer to the VMCPU.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
665 {
666 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the IDT-vectoring information field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 *
681 * @remarks No-long-jump zone!!!
682 */
683DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
684{
685 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
686 {
687 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
688 AssertRCReturn(rc, rc);
689 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
690 }
691 return VINF_SUCCESS;
692}
693
694
695/**
696 * Reads the IDT-vectoring error code from the VMCS into the VMX
697 * transient structure.
698 *
699 * @returns VBox status code.
700 * @param pVmxTransient Pointer to the VMX transient structure.
701 */
702DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
703{
704 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
705 {
706 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
707 AssertRCReturn(rc, rc);
708 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
709 }
710 return VINF_SUCCESS;
711}
712
713
714/**
715 * Enters VMX root mode operation on the current CPU.
716 *
717 * @returns VBox status code.
718 * @param pVM Pointer to the VM (optional, can be NULL, after
719 * a resume).
720 * @param HCPhysCpuPage Physical address of the VMXON region.
721 * @param pvCpuPage Pointer to the VMXON region.
722 */
723static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
724{
725 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
726 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
727 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
728
729 if (pVM)
730 {
731 /* Write the VMCS revision dword to the VMXON region. */
732 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
733 }
734
735 /* Enable the VMX bit in CR4 if necessary. */
736 RTCCUINTREG uCr4 = ASMGetCR4();
737 if (!(uCr4 & X86_CR4_VMXE))
738 ASMSetCR4(uCr4 | X86_CR4_VMXE);
739
740 /* Enter VMX root mode. */
741 int rc = VMXEnable(HCPhysCpuPage);
742 if (RT_FAILURE(rc))
743 ASMSetCR4(uCr4);
744
745 return rc;
746}
747
748
749/**
750 * Exits VMX root mode operation on the current CPU.
751 *
752 * @returns VBox status code.
753 */
754static int hmR0VmxLeaveRootMode(void)
755{
756 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
757
758 /* If we're for some reason not in VMX root mode, then don't leave it. */
759 RTCCUINTREG uHostCR4 = ASMGetCR4();
760 if (uHostCR4 & X86_CR4_VMXE)
761 {
762 /* Exit VMX root mode and clear the VMX bit in CR4. */
763 VMXDisable();
764 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
765 return VINF_SUCCESS;
766 }
767
768 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
769}
770
771
772/**
773 * Allocates and maps one physically contiguous page. The allocated page is
774 * zero'd out. (Used by various VT-x structures).
775 *
776 * @returns IPRT status code.
777 * @param pMemObj Pointer to the ring-0 memory object.
778 * @param ppVirt Where to store the virtual address of the
779 * allocation.
780 * @param pPhys Where to store the physical address of the
781 * allocation.
782 */
783DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
784{
785 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
786 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
787 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
788
789 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
790 if (RT_FAILURE(rc))
791 return rc;
792 *ppVirt = RTR0MemObjAddress(*pMemObj);
793 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
794 ASMMemZero32(*ppVirt, PAGE_SIZE);
795 return VINF_SUCCESS;
796}
797
798
799/**
800 * Frees and unmaps an allocated physical page.
801 *
802 * @param pMemObj Pointer to the ring-0 memory object.
803 * @param ppVirt Where to re-initialize the virtual address of
804 * allocation as 0.
805 * @param pHCPhys Where to re-initialize the physical address of the
806 * allocation as 0.
807 */
808DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
809{
810 AssertPtr(pMemObj);
811 AssertPtr(ppVirt);
812 AssertPtr(pHCPhys);
813 if (*pMemObj != NIL_RTR0MEMOBJ)
814 {
815 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
816 AssertRC(rc);
817 *pMemObj = NIL_RTR0MEMOBJ;
818 *ppVirt = 0;
819 *pHCPhys = 0;
820 }
821}
822
823
824/**
825 * Worker function to free VT-x related structures.
826 *
827 * @returns IPRT status code.
828 * @param pVM Pointer to the VM.
829 */
830static void hmR0VmxStructsFree(PVM pVM)
831{
832 for (VMCPUID i = 0; i < pVM->cCpus; i++)
833 {
834 PVMCPU pVCpu = &pVM->aCpus[i];
835 AssertPtr(pVCpu);
836
837#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
838 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
839 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
840#endif
841
842 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
843 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
844
845 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
846 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
847 }
848
849 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
850#ifdef VBOX_WITH_CRASHDUMP_MAGIC
851 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
852#endif
853}
854
855
856/**
857 * Worker function to allocate VT-x related VM structures.
858 *
859 * @returns IPRT status code.
860 * @param pVM Pointer to the VM.
861 */
862static int hmR0VmxStructsAlloc(PVM pVM)
863{
864 /*
865 * Initialize members up-front so we can cleanup properly on allocation failure.
866 */
867#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
868 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
869 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
870 pVM->hm.s.vmx.HCPhys##a_Name = 0;
871
872#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
873 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
874 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
875 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
876
877#ifdef VBOX_WITH_CRASHDUMP_MAGIC
878 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
879#endif
880 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
881
882 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
883 for (VMCPUID i = 0; i < pVM->cCpus; i++)
884 {
885 PVMCPU pVCpu = &pVM->aCpus[i];
886 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
887 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
888 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
889#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
890 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
891 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
892#endif
893 }
894#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
895#undef VMXLOCAL_INIT_VM_MEMOBJ
896
897 /*
898 * Allocate all the VT-x structures.
899 */
900 int rc = VINF_SUCCESS;
901#ifdef VBOX_WITH_CRASHDUMP_MAGIC
902 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
903 if (RT_FAILURE(rc))
904 goto cleanup;
905 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
906 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
907#endif
908
909 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
910 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
911 {
912 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
913 &pVM->hm.s.vmx.HCPhysApicAccess);
914 if (RT_FAILURE(rc))
915 goto cleanup;
916 }
917
918 /*
919 * Initialize per-VCPU VT-x structures.
920 */
921 for (VMCPUID i = 0; i < pVM->cCpus; i++)
922 {
923 PVMCPU pVCpu = &pVM->aCpus[i];
924 AssertPtr(pVCpu);
925
926 /* Allocate the VM control structure (VMCS). */
927 AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
928 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
929 if (RT_FAILURE(rc))
930 goto cleanup;
931
932 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
933 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
934 {
935 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
936 &pVCpu->hm.s.vmx.HCPhysVirtApic);
937 if (RT_FAILURE(rc))
938 goto cleanup;
939 }
940
941 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
942 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
943 {
944 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
945 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
946 if (RT_FAILURE(rc))
947 goto cleanup;
948 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
949 }
950
951#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
952 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
953 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956
957 /* Allocate the VM-exit MSR-load page for the host MSRs. */
958 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
959 if (RT_FAILURE(rc))
960 goto cleanup;
961#endif
962 }
963
964 return VINF_SUCCESS;
965
966cleanup:
967 hmR0VmxStructsFree(pVM);
968 return rc;
969}
970
971
972/**
973 * Does global VT-x initialization (called during module initialization).
974 *
975 * @returns VBox status code.
976 */
977VMMR0DECL(int) VMXR0GlobalInit(void)
978{
979#ifdef HMVMX_USE_FUNCTION_TABLE
980 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
981# ifdef VBOX_STRICT
982 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
983 Assert(g_apfnVMExitHandlers[i]);
984# endif
985#endif
986 return VINF_SUCCESS;
987}
988
989
990/**
991 * Does global VT-x termination (called during module termination).
992 */
993VMMR0DECL(void) VMXR0GlobalTerm()
994{
995 /* Nothing to do currently. */
996}
997
998
999/**
1000 * Sets up and activates VT-x on the current CPU.
1001 *
1002 * @returns VBox status code.
1003 * @param pCpu Pointer to the global CPU info struct.
1004 * @param pVM Pointer to the VM (can be NULL after a host resume
1005 * operation).
1006 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1007 * fEnabledByHost is true).
1008 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1009 * @a fEnabledByHost is true).
1010 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1011 * enable VT-x on the host.
1012 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1013 */
1014VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1015 void *pvMsrs)
1016{
1017 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1018 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1019 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1020
1021 /* Enable VT-x if it's not already enabled by the host. */
1022 if (!fEnabledByHost)
1023 {
1024 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1025 if (RT_FAILURE(rc))
1026 return rc;
1027 }
1028
1029 /*
1030 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1031 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1032 */
1033 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1034 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1035 {
1036 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1037 pCpu->fFlushAsidBeforeUse = false;
1038 }
1039 else
1040 pCpu->fFlushAsidBeforeUse = true;
1041
1042 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1043 ++pCpu->cTlbFlushes;
1044
1045 return VINF_SUCCESS;
1046}
1047
1048
1049/**
1050 * Deactivates VT-x on the current CPU.
1051 *
1052 * @returns VBox status code.
1053 * @param pCpu Pointer to the global CPU info struct.
1054 * @param pvCpuPage Pointer to the VMXON region.
1055 * @param HCPhysCpuPage Physical address of the VMXON region.
1056 *
1057 * @remarks This function should never be called when SUPR0EnableVTx() or
1058 * similar was used to enable VT-x on the host.
1059 */
1060VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1061{
1062 NOREF(pCpu);
1063 NOREF(pvCpuPage);
1064 NOREF(HCPhysCpuPage);
1065
1066 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1067 return hmR0VmxLeaveRootMode();
1068}
1069
1070
1071/**
1072 * Sets the permission bits for the specified MSR in the MSR bitmap.
1073 *
1074 * @param pVCpu Pointer to the VMCPU.
1075 * @param uMSR The MSR value.
1076 * @param enmRead Whether reading this MSR causes a VM-exit.
1077 * @param enmWrite Whether writing this MSR causes a VM-exit.
1078 */
1079static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1080{
1081 int32_t iBit;
1082 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1083
1084 /*
1085 * Layout:
1086 * 0x000 - 0x3ff - Low MSR read bits
1087 * 0x400 - 0x7ff - High MSR read bits
1088 * 0x800 - 0xbff - Low MSR write bits
1089 * 0xc00 - 0xfff - High MSR write bits
1090 */
1091 if (uMsr <= 0x00001FFF)
1092 iBit = uMsr;
1093 else if ( uMsr >= 0xC0000000
1094 && uMsr <= 0xC0001FFF)
1095 {
1096 iBit = (uMsr - 0xC0000000);
1097 pbMsrBitmap += 0x400;
1098 }
1099 else
1100 {
1101 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1102 return;
1103 }
1104
1105 Assert(iBit <= 0x1fff);
1106 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1107 ASMBitSet(pbMsrBitmap, iBit);
1108 else
1109 ASMBitClear(pbMsrBitmap, iBit);
1110
1111 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1112 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1113 else
1114 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1115}
1116
1117
1118/**
1119 * Flushes the TLB using EPT.
1120 *
1121 * @returns VBox status code.
1122 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1123 * enmFlush).
1124 * @param enmFlush Type of flush.
1125 *
1126 * @remarks Caller is responsible for making sure this function is called only
1127 * when NestedPaging is supported and providing @a enmFlush that is
1128 * supported by the CPU.
1129 */
1130static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1131{
1132 uint64_t au64Descriptor[2];
1133 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1134 au64Descriptor[0] = 0;
1135 else
1136 {
1137 Assert(pVCpu);
1138 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1139 }
1140 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1141
1142 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1143 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1144 rc));
1145 if ( RT_SUCCESS(rc)
1146 && pVCpu)
1147 {
1148 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1149 }
1150}
1151
1152
1153/**
1154 * Flushes the TLB using VPID.
1155 *
1156 * @returns VBox status code.
1157 * @param pVM Pointer to the VM.
1158 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1159 * enmFlush).
1160 * @param enmFlush Type of flush.
1161 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1162 * on @a enmFlush).
1163 */
1164static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1165{
1166 AssertPtr(pVM);
1167 Assert(pVM->hm.s.vmx.fVpid);
1168
1169 uint64_t au64Descriptor[2];
1170 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1171 {
1172 au64Descriptor[0] = 0;
1173 au64Descriptor[1] = 0;
1174 }
1175 else
1176 {
1177 AssertPtr(pVCpu);
1178 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1179 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1180 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1181 au64Descriptor[1] = GCPtr;
1182 }
1183
1184 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1185 AssertMsg(rc == VINF_SUCCESS,
1186 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1187 if ( RT_SUCCESS(rc)
1188 && pVCpu)
1189 {
1190 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1191 }
1192}
1193
1194
1195/**
1196 * Invalidates a guest page by guest virtual address. Only relevant for
1197 * EPT/VPID, otherwise there is nothing really to invalidate.
1198 *
1199 * @returns VBox status code.
1200 * @param pVM Pointer to the VM.
1201 * @param pVCpu Pointer to the VMCPU.
1202 * @param GCVirt Guest virtual address of the page to invalidate.
1203 */
1204VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1205{
1206 AssertPtr(pVM);
1207 AssertPtr(pVCpu);
1208 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1209
1210 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1211 if (!fFlushPending)
1212 {
1213 /*
1214 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1215 * See @bugref{6043} and @bugref{6177}.
1216 *
1217 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1218 * function maybe called in a loop with individual addresses.
1219 */
1220 if (pVM->hm.s.vmx.fVpid)
1221 {
1222 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1223 {
1224 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1225 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1226 }
1227 else
1228 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1229 }
1230 else if (pVM->hm.s.fNestedPaging)
1231 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1232 }
1233
1234 return VINF_SUCCESS;
1235}
1236
1237
1238/**
1239 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1240 * otherwise there is nothing really to invalidate.
1241 *
1242 * @returns VBox status code.
1243 * @param pVM Pointer to the VM.
1244 * @param pVCpu Pointer to the VMCPU.
1245 * @param GCPhys Guest physical address of the page to invalidate.
1246 */
1247VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1248{
1249 LogFlowFunc(("%RGp\n", GCPhys));
1250
1251 /*
1252 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1253 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1254 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1255 */
1256 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1257 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1258 return VINF_SUCCESS;
1259}
1260
1261
1262/**
1263 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1264 * case where neither EPT nor VPID is supported by the CPU.
1265 *
1266 * @param pVM Pointer to the VM.
1267 * @param pVCpu Pointer to the VMCPU.
1268 * @param pCpu Pointer to the global HM struct.
1269 *
1270 * @remarks Called with interrupts disabled.
1271 */
1272static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1273{
1274 AssertPtr(pVCpu);
1275 AssertPtr(pCpu);
1276 NOREF(pVM);
1277
1278 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1279 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1280
1281 pVCpu->hm.s.TlbShootdown.cPages = 0;
1282 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1283 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1284 pVCpu->hm.s.fForceTLBFlush = false;
1285 return;
1286}
1287
1288
1289/**
1290 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1291 *
1292 * @param pVM Pointer to the VM.
1293 * @param pVCpu Pointer to the VMCPU.
1294 * @param pCpu Pointer to the global HM CPU struct.
1295 * @remarks All references to "ASID" in this function pertains to "VPID" in
1296 * Intel's nomenclature. The reason is, to avoid confusion in compare
1297 * statements since the host-CPU copies are named "ASID".
1298 *
1299 * @remarks Called with interrupts disabled.
1300 */
1301static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1302{
1303#ifdef VBOX_WITH_STATISTICS
1304 bool fTlbFlushed = false;
1305# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1306# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1307 if (!fTlbFlushed) \
1308 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1309 } while (0)
1310#else
1311# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1312# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1313#endif
1314
1315 AssertPtr(pVM);
1316 AssertPtr(pCpu);
1317 AssertPtr(pVCpu);
1318 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1319 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1320 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1321
1322
1323 /*
1324 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1325 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1326 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1327 */
1328 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1329 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1330 {
1331 ++pCpu->uCurrentAsid;
1332 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1333 {
1334 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1335 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1336 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1337 }
1338
1339 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1340 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1341 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1342
1343 /*
1344 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1345 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1346 */
1347 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1348 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1349 HMVMX_SET_TAGGED_TLB_FLUSHED();
1350 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1351 }
1352
1353 /* Check for explicit TLB shootdowns. */
1354 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1355 {
1356 /*
1357 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1358 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1359 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1360 * but not guest-physical mappings.
1361 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1362 */
1363 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1364 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1365 HMVMX_SET_TAGGED_TLB_FLUSHED();
1366 }
1367
1368 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1369 * not be executed. See hmQueueInvlPage() where it is commented
1370 * out. Support individual entry flushing someday. */
1371 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1372 {
1373 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1374
1375 /*
1376 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1377 * as supported by the CPU.
1378 */
1379 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1380 {
1381 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1382 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1383 }
1384 else
1385 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1386
1387 HMVMX_SET_TAGGED_TLB_FLUSHED();
1388 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1389 }
1390
1391 pVCpu->hm.s.TlbShootdown.cPages = 0;
1392 pVCpu->hm.s.fForceTLBFlush = false;
1393
1394 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1395
1396 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1397 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1398 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1399 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1400 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1401 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1402 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1403 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1404
1405 /* Update VMCS with the VPID. */
1406 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1407 AssertRC(rc);
1408
1409#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1410}
1411
1412
1413/**
1414 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1415 *
1416 * @returns VBox status code.
1417 * @param pVM Pointer to the VM.
1418 * @param pVCpu Pointer to the VMCPU.
1419 * @param pCpu Pointer to the global HM CPU struct.
1420 *
1421 * @remarks Called with interrupts disabled.
1422 */
1423static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1424{
1425 AssertPtr(pVM);
1426 AssertPtr(pVCpu);
1427 AssertPtr(pCpu);
1428 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1429 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1430
1431 /*
1432 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1433 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1434 */
1435 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1436 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1437 {
1438 pVCpu->hm.s.fForceTLBFlush = true;
1439 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1440 }
1441
1442 /* Check for explicit TLB shootdown flushes. */
1443 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1444 {
1445 pVCpu->hm.s.fForceTLBFlush = true;
1446 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1447 }
1448
1449 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1450 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1451
1452 if (pVCpu->hm.s.fForceTLBFlush)
1453 {
1454 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1455 pVCpu->hm.s.fForceTLBFlush = false;
1456 }
1457 else
1458 {
1459 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1460 * not be executed. See hmQueueInvlPage() where it is commented
1461 * out. Support individual entry flushing someday. */
1462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1463 {
1464 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1465 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1466 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1467 }
1468 else
1469 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1470 }
1471
1472 pVCpu->hm.s.TlbShootdown.cPages = 0;
1473 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1474}
1475
1476
1477/**
1478 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1479 *
1480 * @returns VBox status code.
1481 * @param pVM Pointer to the VM.
1482 * @param pVCpu Pointer to the VMCPU.
1483 * @param pCpu Pointer to the global HM CPU struct.
1484 *
1485 * @remarks Called with interrupts disabled.
1486 */
1487static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1488{
1489 AssertPtr(pVM);
1490 AssertPtr(pVCpu);
1491 AssertPtr(pCpu);
1492 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1493 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1494
1495 /*
1496 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1497 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1498 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1499 */
1500 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1501 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1502 {
1503 pVCpu->hm.s.fForceTLBFlush = true;
1504 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1505 }
1506
1507 /* Check for explicit TLB shootdown flushes. */
1508 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1509 {
1510 /*
1511 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1512 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1513 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1514 */
1515 pVCpu->hm.s.fForceTLBFlush = true;
1516 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1517 }
1518
1519 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1520 if (pVCpu->hm.s.fForceTLBFlush)
1521 {
1522 ++pCpu->uCurrentAsid;
1523 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1524 {
1525 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1526 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1527 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1528 }
1529
1530 pVCpu->hm.s.fForceTLBFlush = false;
1531 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1532 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1533 if (pCpu->fFlushAsidBeforeUse)
1534 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1535 }
1536 else
1537 {
1538 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1539 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1540 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1541 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1542
1543 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1544 * not be executed. See hmQueueInvlPage() where it is commented
1545 * out. Support individual entry flushing someday. */
1546 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1547 {
1548 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1549 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1550 {
1551 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1552 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1553 }
1554 else
1555 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1556 }
1557 else
1558 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1559 }
1560
1561 pVCpu->hm.s.TlbShootdown.cPages = 0;
1562 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1563
1564 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1565 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1566 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1567 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1568 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1569 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1570
1571 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1572 AssertRC(rc);
1573}
1574
1575
1576/**
1577 * Flushes the guest TLB entry based on CPU capabilities.
1578 *
1579 * @param pVCpu Pointer to the VMCPU.
1580 * @param pCpu Pointer to the global HM CPU struct.
1581 */
1582DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1583{
1584 PVM pVM = pVCpu->CTX_SUFF(pVM);
1585 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1586 {
1587 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1588 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1589 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1590 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1591 default:
1592 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1593 break;
1594 }
1595}
1596
1597
1598/**
1599 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1600 * TLB entries from the host TLB before VM-entry.
1601 *
1602 * @returns VBox status code.
1603 * @param pVM Pointer to the VM.
1604 */
1605static int hmR0VmxSetupTaggedTlb(PVM pVM)
1606{
1607 /*
1608 * Determine optimal flush type for Nested Paging.
1609 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1610 * guest execution (see hmR3InitFinalizeR0()).
1611 */
1612 if (pVM->hm.s.fNestedPaging)
1613 {
1614 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1615 {
1616 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1617 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1618 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1619 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1620 else
1621 {
1622 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1623 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1624 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1625 }
1626
1627 /* Make sure the write-back cacheable memory type for EPT is supported. */
1628 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1629 {
1630 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1631 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1632 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1633 }
1634 }
1635 else
1636 {
1637 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1638 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1639 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1640 }
1641 }
1642
1643 /*
1644 * Determine optimal flush type for VPID.
1645 */
1646 if (pVM->hm.s.vmx.fVpid)
1647 {
1648 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1649 {
1650 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1651 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1652 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1653 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1654 else
1655 {
1656 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1657 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1658 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1659 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1660 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1661 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1662 pVM->hm.s.vmx.fVpid = false;
1663 }
1664 }
1665 else
1666 {
1667 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1668 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1669 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1670 pVM->hm.s.vmx.fVpid = false;
1671 }
1672 }
1673
1674 /*
1675 * Setup the handler for flushing tagged-TLBs.
1676 */
1677 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1678 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1679 else if (pVM->hm.s.fNestedPaging)
1680 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1681 else if (pVM->hm.s.vmx.fVpid)
1682 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1683 else
1684 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1685 return VINF_SUCCESS;
1686}
1687
1688
1689/**
1690 * Sets up pin-based VM-execution controls in the VMCS.
1691 *
1692 * @returns VBox status code.
1693 * @param pVM Pointer to the VM.
1694 * @param pVCpu Pointer to the VMCPU.
1695 */
1696static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1697{
1698 AssertPtr(pVM);
1699 AssertPtr(pVCpu);
1700
1701 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1702 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1703
1704 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1705 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1706 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1707
1708 /* Enable the VMX preemption timer. */
1709 if (pVM->hm.s.vmx.fUsePreemptTimer)
1710 {
1711 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1712 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1713 }
1714
1715 if ((val & zap) != val)
1716 {
1717 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1718 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1719 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1720 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1721 }
1722
1723 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1724 AssertRCReturn(rc, rc);
1725
1726 /* Update VCPU with the currently set pin-based VM-execution controls. */
1727 pVCpu->hm.s.vmx.u32PinCtls = val;
1728 return rc;
1729}
1730
1731
1732/**
1733 * Sets up processor-based VM-execution controls in the VMCS.
1734 *
1735 * @returns VBox status code.
1736 * @param pVM Pointer to the VM.
1737 * @param pVMCPU Pointer to the VMCPU.
1738 */
1739static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1740{
1741 AssertPtr(pVM);
1742 AssertPtr(pVCpu);
1743
1744 int rc = VERR_INTERNAL_ERROR_5;
1745 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1746 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1747
1748 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1749 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1750 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1751 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1752 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1753 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1754 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1755
1756 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1757 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1758 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1759 {
1760 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1761 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1762 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1763 }
1764
1765 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1766 if (!pVM->hm.s.fNestedPaging)
1767 {
1768 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1769 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1770 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1771 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1772 }
1773
1774 /* Use TPR shadowing if supported by the CPU. */
1775 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1776 {
1777 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1778 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1779 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1780 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1781 AssertRCReturn(rc, rc);
1782
1783 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1784 /* CR8 writes causes a VM-exit based on TPR threshold. */
1785 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1786 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1787 }
1788 else
1789 {
1790 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1791 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1792 }
1793
1794 /* Use MSR-bitmaps if supported by the CPU. */
1795 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1796 {
1797 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1798
1799 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1800 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1801 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1802 AssertRCReturn(rc, rc);
1803
1804 /*
1805 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1806 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1807 */
1808 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1809 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1810 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1811 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1812 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1813 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1814 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1815 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1816 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1817 }
1818
1819 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1820 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1821 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1822
1823 if ((val & zap) != val)
1824 {
1825 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1826 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1827 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1828 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1829 }
1830
1831 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1832 AssertRCReturn(rc, rc);
1833
1834 /* Update VCPU with the currently set processor-based VM-execution controls. */
1835 pVCpu->hm.s.vmx.u32ProcCtls = val;
1836
1837 /*
1838 * Secondary processor-based VM-execution controls.
1839 */
1840 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1841 {
1842 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1843 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1844
1845 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1846 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1847
1848 if (pVM->hm.s.fNestedPaging)
1849 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1850 else
1851 {
1852 /*
1853 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1854 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1855 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1856 */
1857 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1858 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1859 }
1860
1861 if (pVM->hm.s.vmx.fVpid)
1862 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1863
1864 if (pVM->hm.s.vmx.fUnrestrictedGuest)
1865 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1866
1867 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1868 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1869 * done dynamically. */
1870 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1871 {
1872 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1873 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1874 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1875 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1876 AssertRCReturn(rc, rc);
1877 }
1878
1879 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1880 {
1881 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1882 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1883 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1884 }
1885
1886 if ((val & zap) != val)
1887 {
1888 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1889 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1890 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1891 }
1892
1893 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1894 AssertRCReturn(rc, rc);
1895
1896 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1897 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1898 }
1899 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1900 {
1901 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1902 "available\n"));
1903 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1904 }
1905
1906 return VINF_SUCCESS;
1907}
1908
1909
1910/**
1911 * Sets up miscellaneous (everything other than Pin & Processor-based
1912 * VM-execution) control fields in the VMCS.
1913 *
1914 * @returns VBox status code.
1915 * @param pVM Pointer to the VM.
1916 * @param pVCpu Pointer to the VMCPU.
1917 */
1918static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1919{
1920 AssertPtr(pVM);
1921 AssertPtr(pVCpu);
1922
1923 int rc = VERR_GENERAL_FAILURE;
1924
1925 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1926#if 0
1927 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1928 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1929 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1930
1931 /*
1932 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1933 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1934 * We thus use the exception bitmap to control it rather than use both.
1935 */
1936 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1937 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1938
1939 /** @todo Explore possibility of using IO-bitmaps. */
1940 /* All IO & IOIO instructions cause VM-exits. */
1941 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1942 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1943
1944 /* Initialize the MSR-bitmap area. */
1945 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1946 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1947 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1948#endif
1949
1950#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1951 /* Setup MSR autoloading/storing. */
1952 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1953 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1954 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1955 AssertRCReturn(rc, rc);
1956 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1957 AssertRCReturn(rc, rc);
1958
1959 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1960 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1961 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1962 AssertRCReturn(rc, rc);
1963#endif
1964
1965 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1966 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1967 AssertRCReturn(rc, rc);
1968
1969 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1970#if 0
1971 /* Setup debug controls */
1972 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1973 AssertRCReturn(rc, rc);
1974 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1975 AssertRCReturn(rc, rc);
1976#endif
1977
1978 return rc;
1979}
1980
1981
1982/**
1983 * Sets up the initial exception bitmap in the VMCS based on static conditions
1984 * (i.e. conditions that cannot ever change after starting the VM).
1985 *
1986 * @returns VBox status code.
1987 * @param pVM Pointer to the VM.
1988 * @param pVCpu Pointer to the VMCPU.
1989 */
1990static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1991{
1992 AssertPtr(pVM);
1993 AssertPtr(pVCpu);
1994
1995 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1996
1997 uint32_t u32XcptBitmap = 0;
1998
1999 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2000 if (!pVM->hm.s.fNestedPaging)
2001 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2002
2003 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2004 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2005 AssertRCReturn(rc, rc);
2006 return rc;
2007}
2008
2009
2010/**
2011 * Sets up the initial guest-state mask. The guest-state mask is consulted
2012 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2013 * for the nested virtualization case (as it would cause a VM-exit).
2014 *
2015 * @param pVCpu Pointer to the VMCPU.
2016 */
2017static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2018{
2019 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2020 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2021 return VINF_SUCCESS;
2022}
2023
2024
2025/**
2026 * Does per-VM VT-x initialization.
2027 *
2028 * @returns VBox status code.
2029 * @param pVM Pointer to the VM.
2030 */
2031VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2032{
2033 LogFlowFunc(("pVM=%p\n", pVM));
2034
2035 int rc = hmR0VmxStructsAlloc(pVM);
2036 if (RT_FAILURE(rc))
2037 {
2038 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2039 return rc;
2040 }
2041
2042 return VINF_SUCCESS;
2043}
2044
2045
2046/**
2047 * Does per-VM VT-x termination.
2048 *
2049 * @returns VBox status code.
2050 * @param pVM Pointer to the VM.
2051 */
2052VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2053{
2054 LogFlowFunc(("pVM=%p\n", pVM));
2055
2056#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2057 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2058 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2059#endif
2060 hmR0VmxStructsFree(pVM);
2061 return VINF_SUCCESS;
2062}
2063
2064
2065/**
2066 * Sets up the VM for execution under VT-x.
2067 * This function is only called once per-VM during initialization.
2068 *
2069 * @returns VBox status code.
2070 * @param pVM Pointer to the VM.
2071 */
2072VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2073{
2074 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2075 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2076
2077 LogFlowFunc(("pVM=%p\n", pVM));
2078
2079 /*
2080 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2081 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2082 */
2083 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2084 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2085 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2086 || !pVM->hm.s.vmx.pRealModeTSS))
2087 {
2088 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2089 return VERR_INTERNAL_ERROR;
2090 }
2091
2092#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2093 /*
2094 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2095 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2096 */
2097 if ( pVM->hm.s.fAllow64BitGuests
2098 && !HMVMX_IS_64BIT_HOST_MODE())
2099 {
2100 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2101 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2102 }
2103#endif
2104
2105 /* Initialize these always, see hmR3InitFinalizeR0().*/
2106 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2107 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2108
2109 /* Setup the tagged-TLB flush handlers. */
2110 int rc = hmR0VmxSetupTaggedTlb(pVM);
2111 if (RT_FAILURE(rc))
2112 {
2113 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2114 return rc;
2115 }
2116
2117 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2118 {
2119 PVMCPU pVCpu = &pVM->aCpus[i];
2120 AssertPtr(pVCpu);
2121 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2122
2123 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2124 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2125
2126 /* Set revision dword at the beginning of the VMCS structure. */
2127 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2128
2129 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2130 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2131 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2132 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2133
2134 /* Load this VMCS as the current VMCS. */
2135 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2136 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2137 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2138
2139 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2140 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2141 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2142
2143 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2144 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2145 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2146
2147 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2148 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2149 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2150
2151 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2152 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2153 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2154
2155 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2156 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2157 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2158
2159#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2160 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2161 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2162 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2163#endif
2164
2165 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2166 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2167 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2168 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2169
2170 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2171
2172 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2173 }
2174
2175 return VINF_SUCCESS;
2176}
2177
2178
2179/**
2180 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2181 * the VMCS.
2182 *
2183 * @returns VBox status code.
2184 * @param pVM Pointer to the VM.
2185 * @param pVCpu Pointer to the VMCPU.
2186 */
2187DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2188{
2189 RTCCUINTREG uReg = ASMGetCR0();
2190 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2191 AssertRCReturn(rc, rc);
2192
2193#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2194 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2195 if (HMVMX_IS_64BIT_HOST_MODE())
2196 {
2197 uint64_t uRegCR3 = HMR0Get64bitCR3();
2198 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2199 }
2200 else
2201#endif
2202 {
2203 uReg = ASMGetCR3();
2204 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2205 }
2206 AssertRCReturn(rc, rc);
2207
2208 uReg = ASMGetCR4();
2209 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2210 AssertRCReturn(rc, rc);
2211 return rc;
2212}
2213
2214
2215/**
2216 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2217 * the host-state area in the VMCS.
2218 *
2219 * @returns VBox status code.
2220 * @param pVM Pointer to the VM.
2221 * @param pVCpu Pointer to the VMCPU.
2222 */
2223DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2224{
2225 int rc = VERR_INTERNAL_ERROR_5;
2226
2227 /*
2228 * Host DS, ES, FS and GS segment registers.
2229 */
2230#if HC_ARCH_BITS == 64
2231 RTSEL uSelDS = ASMGetDS();
2232 RTSEL uSelES = ASMGetES();
2233 RTSEL uSelFS = ASMGetFS();
2234 RTSEL uSelGS = ASMGetGS();
2235#else
2236 RTSEL uSelDS = 0;
2237 RTSEL uSelES = 0;
2238 RTSEL uSelFS = 0;
2239 RTSEL uSelGS = 0;
2240#endif
2241
2242 /* Recalculate which host-state bits need to be manually restored. */
2243 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2244
2245 /*
2246 * Host CS and SS segment registers.
2247 */
2248#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2249 RTSEL uSelCS;
2250 RTSEL uSelSS;
2251 if (HMVMX_IS_64BIT_HOST_MODE())
2252 {
2253 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2254 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2255 }
2256 else
2257 {
2258 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2259 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2260 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2261 }
2262#else
2263 RTSEL uSelCS = ASMGetCS();
2264 RTSEL uSelSS = ASMGetSS();
2265#endif
2266
2267 /*
2268 * Host TR segment register.
2269 */
2270 RTSEL uSelTR = ASMGetTR();
2271
2272#if HC_ARCH_BITS == 64
2273 /*
2274 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2275 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2276 */
2277 if (uSelDS & (X86_SEL_RPL | X86_SEL_LDT))
2278 {
2279 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_DS;
2280 pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2281 uSelDS = 0;
2282 }
2283 if (uSelES & (X86_SEL_RPL | X86_SEL_LDT))
2284 {
2285 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_ES;
2286 pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2287 uSelES = 0;
2288 }
2289 if (uSelFS & (X86_SEL_RPL | X86_SEL_LDT))
2290 {
2291 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_FS;
2292 pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2293 uSelFS = 0;
2294 }
2295 if (uSelGS & (X86_SEL_RPL | X86_SEL_LDT))
2296 {
2297 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_GS;
2298 pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2299 uSelGS = 0;
2300 }
2301#endif
2302
2303 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2304 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2305 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2306 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2307 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2308 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2309 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2310 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2311 Assert(uSelCS);
2312 Assert(uSelTR);
2313
2314 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2315#if 0
2316 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2317 Assert(uSelSS != 0);
2318#endif
2319
2320 /* Write these host selector fields into the host-state area in the VMCS. */
2321 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2322 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2323#if HC_ARCH_BITS == 64
2324 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2325 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2326 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2327 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2328#endif
2329 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2330
2331 /*
2332 * Host GDTR and IDTR.
2333 */
2334 RTGDTR Gdtr;
2335 RT_ZERO(Gdtr);
2336#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2337 if (HMVMX_IS_64BIT_HOST_MODE())
2338 {
2339 X86XDTR64 Gdtr64;
2340 X86XDTR64 Idtr64;
2341 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2342 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2343 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2344
2345 Gdtr.cbGdt = Gdtr64.cb;
2346 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2347 }
2348 else
2349#endif
2350 {
2351 RTIDTR Idtr;
2352 ASMGetGDTR(&Gdtr);
2353 ASMGetIDTR(&Idtr);
2354 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2355 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2356
2357#if HC_ARCH_BITS == 64
2358 /*
2359 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2360 * maximum limit (0xffff) on every VM-exit.
2361 */
2362 if (Gdtr.cbGdt != 0xffff)
2363 {
2364 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2365 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2366 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2367 }
2368
2369 /*
2370 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2371 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2372 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2373 */
2374 if (Idtr.cbIdt < 0x0fff)
2375 {
2376 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2377 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2378 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2379 }
2380#endif
2381 }
2382
2383 /*
2384 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2385 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2386 */
2387 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2388 {
2389 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2390 return VERR_VMX_INVALID_HOST_STATE;
2391 }
2392
2393 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2394#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2395 if (HMVMX_IS_64BIT_HOST_MODE())
2396 {
2397 /* We need the 64-bit TR base for hybrid darwin. */
2398 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2399 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2400 }
2401 else
2402#endif
2403 {
2404 uintptr_t uTRBase;
2405#if HC_ARCH_BITS == 64
2406 uTRBase = X86DESC64_BASE(pDesc);
2407
2408 /*
2409 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2410 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2411 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2412 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2413 *
2414 * [1] See Intel spec. 3.5 "System Descriptor Types".
2415 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2416 */
2417 Assert(pDesc->System.u4Type == 11);
2418 if ( pDesc->System.u16LimitLow != 0x67
2419 || pDesc->System.u4LimitHigh)
2420 {
2421 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2422 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2423
2424 /* Store the GDTR here as we need it while restoring TR. */
2425 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2426 }
2427#else
2428 uTRBase = X86DESC_BASE(pDesc);
2429#endif
2430 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2431 }
2432 AssertRCReturn(rc, rc);
2433
2434 /*
2435 * Host FS base and GS base.
2436 */
2437#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2438 if (HMVMX_IS_64BIT_HOST_MODE())
2439 {
2440 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2441 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2442 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2443 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2444
2445# if HC_ARCH_BITS == 64
2446 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2447 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2448 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2449 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2450 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2451# endif
2452 }
2453#endif
2454 return rc;
2455}
2456
2457
2458/**
2459 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2460 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2461 * the host after every successful VM exit.
2462 *
2463 * @returns VBox status code.
2464 * @param pVM Pointer to the VM.
2465 * @param pVCpu Pointer to the VMCPU.
2466 */
2467DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2468{
2469 AssertPtr(pVCpu);
2470 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2471
2472 int rc = VINF_SUCCESS;
2473#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2474 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2475 uint32_t cHostMsrs = 0;
2476 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2477
2478 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2479 {
2480 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2481
2482# if HC_ARCH_BITS == 64
2483 /* Paranoia. 64-bit code requires these bits to be set always. */
2484 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2485
2486 /*
2487 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2488 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2489 * some reason (e.g. allow transparent reads) we would activate the code below.
2490 */
2491# if 0
2492 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2493 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2494 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2495 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2496 if (CPUMIsGuestInLongMode(pVCpu))
2497 {
2498 uint64_t u64GuestEfer;
2499 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2500 AssertRC(rc);
2501
2502 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2503 {
2504 pHostMsr->u32Msr = MSR_K6_EFER;
2505 pHostMsr->u32Reserved = 0;
2506 pHostMsr->u64Value = u64HostEfer;
2507 pHostMsr++; cHostMsrs++;
2508 }
2509 }
2510# endif
2511# else /* HC_ARCH_BITS != 64 */
2512 pHostMsr->u32Msr = MSR_K6_EFER;
2513 pHostMsr->u32Reserved = 0;
2514# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2515 if (CPUMIsGuestInLongMode(pVCpu))
2516 {
2517 /* Must match the EFER value in our 64 bits switcher. */
2518 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2519 }
2520 else
2521# endif
2522 pHostMsr->u64Value = u64HostEfer;
2523 pHostMsr++; cHostMsrs++;
2524# endif /* HC_ARCH_BITS == 64 */
2525 }
2526
2527# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2528 if (HMVMX_IS_64BIT_HOST_MODE())
2529 {
2530 pHostMsr->u32Msr = MSR_K6_STAR;
2531 pHostMsr->u32Reserved = 0;
2532 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2533 pHostMsr++; cHostMsrs++;
2534 pHostMsr->u32Msr = MSR_K8_LSTAR;
2535 pHostMsr->u32Reserved = 0;
2536 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2537 pHostMsr++; cHostMsrs++;
2538 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2539 pHostMsr->u32Reserved = 0;
2540 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2541 pHostMsr++; cHostMsrs++;
2542 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2543 pHostMsr->u32Reserved = 0;
2544 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2545 pHostMsr++; cHostMsrs++;
2546 }
2547# endif
2548
2549 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2550 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2551 {
2552 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2553 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2554 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2555 }
2556
2557 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2558#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2559
2560 /*
2561 * Host Sysenter MSRs.
2562 */
2563 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2564 AssertRCReturn(rc, rc);
2565#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2566 if (HMVMX_IS_64BIT_HOST_MODE())
2567 {
2568 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2569 AssertRCReturn(rc, rc);
2570 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2571 }
2572 else
2573 {
2574 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2575 AssertRCReturn(rc, rc);
2576 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2577 }
2578#elif HC_ARCH_BITS == 32
2579 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2580 AssertRCReturn(rc, rc);
2581 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2582#else
2583 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2584 AssertRCReturn(rc, rc);
2585 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2586#endif
2587 AssertRCReturn(rc, rc);
2588
2589 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2590 * hmR0VmxSetupExitCtls() !! */
2591 return rc;
2592}
2593
2594
2595/**
2596 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2597 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2598 * controls".
2599 *
2600 * @returns VBox status code.
2601 * @param pVCpu Pointer to the VMCPU.
2602 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2603 * out-of-sync. Make sure to update the required fields
2604 * before using them.
2605 *
2606 * @remarks No-long-jump zone!!!
2607 */
2608DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2609{
2610 int rc = VINF_SUCCESS;
2611 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2612 {
2613 PVM pVM = pVCpu->CTX_SUFF(pVM);
2614 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2615 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2616
2617 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2618 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2619
2620 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2621 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2622 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2623 else
2624 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2625
2626 /*
2627 * The following should not be set (since we're not in SMM mode):
2628 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2629 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2630 */
2631
2632 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2633 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2634 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2635
2636 if ((val & zap) != val)
2637 {
2638 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2639 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2640 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2641 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2642 }
2643
2644 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2645 AssertRCReturn(rc, rc);
2646
2647 /* Update VCPU with the currently set VM-exit controls. */
2648 pVCpu->hm.s.vmx.u32EntryCtls = val;
2649 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2650 }
2651 return rc;
2652}
2653
2654
2655/**
2656 * Sets up the VM-exit controls in the VMCS.
2657 *
2658 * @returns VBox status code.
2659 * @param pVM Pointer to the VM.
2660 * @param pVCpu Pointer to the VMCPU.
2661 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2662 * out-of-sync. Make sure to update the required fields
2663 * before using them.
2664 *
2665 * @remarks requires EFER.
2666 */
2667DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2668{
2669 int rc = VINF_SUCCESS;
2670 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2671 {
2672 PVM pVM = pVCpu->CTX_SUFF(pVM);
2673 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2674 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2675
2676 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2677 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2678
2679 /*
2680 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2681 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2682 */
2683#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2684 if (HMVMX_IS_64BIT_HOST_MODE())
2685 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2686 else
2687 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2688#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2689 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2690 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2691 else
2692 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2693#endif
2694
2695 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2696 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2697
2698 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2699 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2700 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2701 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2702 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2703
2704 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2705 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2706
2707 if ((val & zap) != val)
2708 {
2709 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2710 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2711 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2712 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2713 }
2714
2715 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2716 AssertRCReturn(rc, rc);
2717
2718 /* Update VCPU with the currently set VM-exit controls. */
2719 pVCpu->hm.s.vmx.u32ExitCtls = val;
2720 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2721 }
2722 return rc;
2723}
2724
2725
2726/**
2727 * Loads the guest APIC and related state.
2728 *
2729 * @returns VBox status code.
2730 * @param pVM Pointer to the VM.
2731 * @param pVCpu Pointer to the VMCPU.
2732 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2733 * out-of-sync. Make sure to update the required fields
2734 * before using them.
2735 */
2736DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2737{
2738 int rc = VINF_SUCCESS;
2739 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2740 {
2741 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2742 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2743 {
2744 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2745
2746 bool fPendingIntr = false;
2747 uint8_t u8Tpr = 0;
2748 uint8_t u8PendingIntr = 0;
2749 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2750 AssertRCReturn(rc, rc);
2751
2752 /*
2753 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2754 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2755 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2756 * the interrupt when we VM-exit for other reasons.
2757 */
2758 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2759 uint32_t u32TprThreshold = 0;
2760 if (fPendingIntr)
2761 {
2762 /* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2763 const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2764 const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2765 if (u8PendingPriority <= u8TprPriority)
2766 u32TprThreshold = u8PendingPriority;
2767 else
2768 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2769 }
2770 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2771
2772 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2773 AssertRCReturn(rc, rc);
2774 }
2775
2776 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2777 }
2778 return rc;
2779}
2780
2781
2782/**
2783 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2784 *
2785 * @returns Guest's interruptibility-state.
2786 * @param pVCpu Pointer to the VMCPU.
2787 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2788 * out-of-sync. Make sure to update the required fields
2789 * before using them.
2790 *
2791 * @remarks No-long-jump zone!!!
2792 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2793 */
2794DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2795{
2796 /*
2797 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2798 * inhibit interrupts or clear any existing interrupt-inhibition.
2799 */
2800 uint32_t uIntrState = 0;
2801 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2802 {
2803 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2804 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
2805 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2806 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2807 {
2808 /*
2809 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2810 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2811 */
2812 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2813 }
2814 else if (pMixedCtx->eflags.Bits.u1IF)
2815 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2816 else
2817 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2818 }
2819 return uIntrState;
2820}
2821
2822
2823/**
2824 * Loads the guest's interruptibility-state into the guest-state area in the
2825 * VMCS.
2826 *
2827 * @returns VBox status code.
2828 * @param pVCpu Pointer to the VMCPU.
2829 * @param uIntrState The interruptibility-state to set.
2830 */
2831static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2832{
2833 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2834 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2835 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2836 AssertRCReturn(rc, rc);
2837 return rc;
2838}
2839
2840
2841/**
2842 * Loads the guest's RIP into the guest-state area in the VMCS.
2843 *
2844 * @returns VBox status code.
2845 * @param pVCpu Pointer to the VMCPU.
2846 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2847 * out-of-sync. Make sure to update the required fields
2848 * before using them.
2849 *
2850 * @remarks No-long-jump zone!!!
2851 */
2852static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2853{
2854 int rc = VINF_SUCCESS;
2855 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2856 {
2857 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2858 AssertRCReturn(rc, rc);
2859 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2860 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#x\n", pMixedCtx->rip, pVCpu->hm.s.fContextUseFlags));
2861 }
2862 return rc;
2863}
2864
2865
2866/**
2867 * Loads the guest's RSP into the guest-state area in the VMCS.
2868 *
2869 * @returns VBox status code.
2870 * @param pVCpu Pointer to the VMCPU.
2871 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2872 * out-of-sync. Make sure to update the required fields
2873 * before using them.
2874 *
2875 * @remarks No-long-jump zone!!!
2876 */
2877static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2878{
2879 int rc = VINF_SUCCESS;
2880 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2881 {
2882 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2883 AssertRCReturn(rc, rc);
2884 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2885 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2886 }
2887 return rc;
2888}
2889
2890
2891/**
2892 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
2893 *
2894 * @returns VBox status code.
2895 * @param pVCpu Pointer to the VMCPU.
2896 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2897 * out-of-sync. Make sure to update the required fields
2898 * before using them.
2899 *
2900 * @remarks No-long-jump zone!!!
2901 */
2902static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2903{
2904 int rc = VINF_SUCCESS;
2905 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2906 {
2907 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2908 Let us assert it as such and use 32-bit VMWRITE. */
2909 Assert(!(pMixedCtx->rflags.u64 >> 32));
2910 X86EFLAGS Eflags = pMixedCtx->eflags;
2911 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2912 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2913
2914 /*
2915 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2916 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2917 */
2918 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2919 {
2920 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2921 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2922 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2923 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2924 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2925 }
2926
2927 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2928 AssertRCReturn(rc, rc);
2929
2930 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2931 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2932 }
2933 return rc;
2934}
2935
2936
2937/**
2938 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2939 *
2940 * @returns VBox status code.
2941 * @param pVCpu Pointer to the VMCPU.
2942 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2943 * out-of-sync. Make sure to update the required fields
2944 * before using them.
2945 *
2946 * @remarks No-long-jump zone!!!
2947 */
2948DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2949{
2950 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2951 AssertRCReturn(rc, rc);
2952 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2953 AssertRCReturn(rc, rc);
2954 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2955 AssertRCReturn(rc, rc);
2956 return rc;
2957}
2958
2959
2960/**
2961 * Loads the guest CR0 control register into the guest-state area in the VMCS.
2962 * CR0 is partially shared with the host and we have to consider the FPU bits.
2963 *
2964 * @returns VBox status code.
2965 * @param pVM Pointer to the VM.
2966 * @param pVCpu Pointer to the VMCPU.
2967 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2968 * out-of-sync. Make sure to update the required fields
2969 * before using them.
2970 *
2971 * @remarks No-long-jump zone!!!
2972 */
2973static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2974{
2975 /*
2976 * Guest CR0.
2977 * Guest FPU.
2978 */
2979 int rc = VINF_SUCCESS;
2980 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2981 {
2982 Assert(!(pMixedCtx->cr0 >> 32));
2983 uint32_t u32GuestCR0 = pMixedCtx->cr0;
2984 PVM pVM = pVCpu->CTX_SUFF(pVM);
2985
2986 /* The guest's view (read access) of its CR0 is unblemished. */
2987 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2988 AssertRCReturn(rc, rc);
2989 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2990
2991 /* Setup VT-x's view of the guest CR0. */
2992 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2993 if (pVM->hm.s.fNestedPaging)
2994 {
2995 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
2996 {
2997 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2998 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2999 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3000 }
3001 else
3002 {
3003 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3004 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3005 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3006 }
3007
3008 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3009 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3010 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3011
3012 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3013 AssertRCReturn(rc, rc);
3014 }
3015 else
3016 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3017
3018 /*
3019 * Guest FPU bits.
3020 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3021 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3022 */
3023 u32GuestCR0 |= X86_CR0_NE;
3024 bool fInterceptNM = false;
3025 if (CPUMIsGuestFPUStateActive(pVCpu))
3026 {
3027 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3028 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3029 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3030 }
3031 else
3032 {
3033 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3034 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3035 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3036 }
3037
3038 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3039 bool fInterceptMF = false;
3040 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3041 fInterceptMF = true;
3042
3043 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3044 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3045 {
3046 Assert(PDMVmmDevHeapIsEnabled(pVM));
3047 Assert(pVM->hm.s.vmx.pRealModeTSS);
3048 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3049 fInterceptNM = true;
3050 fInterceptMF = true;
3051 }
3052 else
3053 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3054
3055 if (fInterceptNM)
3056 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3057 else
3058 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3059
3060 if (fInterceptMF)
3061 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3062 else
3063 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3064
3065 /* Additional intercepts for debugging, define these yourself explicitly. */
3066#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3067 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3068 | RT_BIT(X86_XCPT_BP)
3069 | RT_BIT(X86_XCPT_DB)
3070 | RT_BIT(X86_XCPT_DE)
3071 | RT_BIT(X86_XCPT_NM)
3072 | RT_BIT(X86_XCPT_UD)
3073 | RT_BIT(X86_XCPT_NP)
3074 | RT_BIT(X86_XCPT_SS)
3075 | RT_BIT(X86_XCPT_GP)
3076 | RT_BIT(X86_XCPT_PF)
3077 | RT_BIT(X86_XCPT_MF)
3078 ;
3079#elif defined(HMVMX_ALWAYS_TRAP_PF)
3080 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3081#endif
3082
3083 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3084
3085 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3086 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3087 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3088 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3089 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3090 else
3091 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3092
3093 u32GuestCR0 |= uSetCR0;
3094 u32GuestCR0 &= uZapCR0;
3095 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3096
3097 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3098 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3099 AssertRCReturn(rc, rc);
3100 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3101 AssertRCReturn(rc, rc);
3102 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3103
3104 /*
3105 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3106 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3107 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3108 */
3109 uint32_t u32CR0Mask = 0;
3110 u32CR0Mask = X86_CR0_PE
3111 | X86_CR0_NE
3112 | X86_CR0_WP
3113 | X86_CR0_PG
3114 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3115 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3116 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3117 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3118 u32CR0Mask &= ~X86_CR0_PE;
3119 if (pVM->hm.s.fNestedPaging)
3120 u32CR0Mask &= ~X86_CR0_WP;
3121
3122 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3123 if (fInterceptNM)
3124 u32CR0Mask |= (X86_CR0_TS | X86_CR0_MP);
3125 else
3126 u32CR0Mask &= ~(X86_CR0_TS | X86_CR0_MP);
3127
3128 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3129 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3130 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3131 AssertRCReturn(rc, rc);
3132
3133 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3134 }
3135 return rc;
3136}
3137
3138
3139/**
3140 * Loads the guest control registers (CR3, CR4) into the guest-state area
3141 * in the VMCS.
3142 *
3143 * @returns VBox status code.
3144 * @param pVM Pointer to the VM.
3145 * @param pVCpu Pointer to the VMCPU.
3146 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3147 * out-of-sync. Make sure to update the required fields
3148 * before using them.
3149 *
3150 * @remarks No-long-jump zone!!!
3151 */
3152static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3153{
3154 int rc = VINF_SUCCESS;
3155 PVM pVM = pVCpu->CTX_SUFF(pVM);
3156
3157 /*
3158 * Guest CR2.
3159 * It's always loaded in the assembler code. Nothing to do here.
3160 */
3161
3162 /*
3163 * Guest CR3.
3164 */
3165 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3166 {
3167 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3168 if (pVM->hm.s.fNestedPaging)
3169 {
3170 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3171
3172 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3173 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3174 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3175 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3176
3177 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3178 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3179 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3180
3181 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3182 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3183 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3184 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3185
3186 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3187 AssertRCReturn(rc, rc);
3188 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3189
3190 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3191 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3192 {
3193 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3194 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3195 {
3196 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3197 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3198 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3199 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3200 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3201 }
3202
3203 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3204 have Unrestricted Execution to handle the guest when it's not using paging. */
3205 GCPhysGuestCR3 = pMixedCtx->cr3;
3206 }
3207 else
3208 {
3209 /*
3210 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3211 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3212 * EPT takes care of translating it to host-physical addresses.
3213 */
3214 RTGCPHYS GCPhys;
3215 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3216 Assert(PDMVmmDevHeapIsEnabled(pVM));
3217
3218 /* We obtain it here every time as the guest could have relocated this PCI region. */
3219 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3220 AssertRCReturn(rc, rc);
3221
3222 GCPhysGuestCR3 = GCPhys;
3223 }
3224
3225 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3226 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3227 }
3228 else
3229 {
3230 /* Non-nested paging case, just use the hypervisor's CR3. */
3231 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3232
3233 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3234 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3235 }
3236 AssertRCReturn(rc, rc);
3237
3238 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3239 }
3240
3241 /*
3242 * Guest CR4.
3243 */
3244 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3245 {
3246 Assert(!(pMixedCtx->cr4 >> 32));
3247 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3248
3249 /* The guest's view of its CR4 is unblemished. */
3250 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3251 AssertRCReturn(rc, rc);
3252 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3253
3254 /* Setup VT-x's view of the guest CR4. */
3255 /*
3256 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3257 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3258 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3259 */
3260 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3261 {
3262 Assert(pVM->hm.s.vmx.pRealModeTSS);
3263 Assert(PDMVmmDevHeapIsEnabled(pVM));
3264 u32GuestCR4 &= ~X86_CR4_VME;
3265 }
3266
3267 if (pVM->hm.s.fNestedPaging)
3268 {
3269 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3270 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3271 {
3272 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3273 u32GuestCR4 |= X86_CR4_PSE;
3274 /* Our identity mapping is a 32 bits page directory. */
3275 u32GuestCR4 &= ~X86_CR4_PAE;
3276 }
3277 /* else use guest CR4.*/
3278 }
3279 else
3280 {
3281 /*
3282 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3283 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3284 */
3285 switch (pVCpu->hm.s.enmShadowMode)
3286 {
3287 case PGMMODE_REAL: /* Real-mode. */
3288 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3289 case PGMMODE_32_BIT: /* 32-bit paging. */
3290 {
3291 u32GuestCR4 &= ~X86_CR4_PAE;
3292 break;
3293 }
3294
3295 case PGMMODE_PAE: /* PAE paging. */
3296 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3297 {
3298 u32GuestCR4 |= X86_CR4_PAE;
3299 break;
3300 }
3301
3302 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3303 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3304#ifdef VBOX_ENABLE_64_BITS_GUESTS
3305 break;
3306#endif
3307 default:
3308 AssertFailed();
3309 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3310 }
3311 }
3312
3313 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3314 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3315 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3316 u32GuestCR4 |= uSetCR4;
3317 u32GuestCR4 &= uZapCR4;
3318
3319 /* Write VT-x's view of the guest CR4 into the VMCS. */
3320 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3321 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3322 AssertRCReturn(rc, rc);
3323
3324 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3325 uint32_t u32CR4Mask = 0;
3326 u32CR4Mask = X86_CR4_VME
3327 | X86_CR4_PAE
3328 | X86_CR4_PGE
3329 | X86_CR4_PSE
3330 | X86_CR4_VMXE;
3331 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3332 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3333 AssertRCReturn(rc, rc);
3334
3335 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3336 }
3337 return rc;
3338}
3339
3340
3341/**
3342 * Loads the guest debug registers into the guest-state area in the VMCS.
3343 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3344 *
3345 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3346 *
3347 * @returns VBox status code.
3348 * @param pVCpu Pointer to the VMCPU.
3349 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3350 * out-of-sync. Make sure to update the required fields
3351 * before using them.
3352 *
3353 * @remarks No-long-jump zone!!!
3354 */
3355static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3356{
3357 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3358 return VINF_SUCCESS;
3359
3360#ifdef VBOX_STRICT
3361 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3362 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3363 {
3364 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3365 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3366 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3367 }
3368#endif
3369
3370 int rc;
3371 PVM pVM = pVCpu->CTX_SUFF(pVM);
3372 bool fInterceptDB = false;
3373 bool fInterceptMovDRx = false;
3374 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3375 {
3376 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3377 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3378 {
3379 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3380 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3381 AssertRCReturn(rc, rc);
3382 Assert(fInterceptDB == false);
3383 }
3384 else
3385 {
3386 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3387 pVCpu->hm.s.fClearTrapFlag = true;
3388 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RFLAGS;
3389 fInterceptDB = true;
3390 }
3391 }
3392
3393 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3394 {
3395 /*
3396 * Use the combined guest and host DRx values found in the hypervisor
3397 * register set because the debugger has breakpoints active or someone
3398 * is single stepping on the host side without a monitor trap flag.
3399 *
3400 * Note! DBGF expects a clean DR6 state before executing guest code.
3401 */
3402 if (!CPUMIsHyperDebugStateActive(pVCpu))
3403 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3404 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3405 Assert(CPUMIsHyperDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3406
3407 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3408 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3409 AssertRCReturn(rc, rc);
3410
3411 fInterceptDB = true;
3412 fInterceptMovDRx = true;
3413 }
3414 else
3415 {
3416 /*
3417 * If the guest has enabled debug registers, we need to load them prior to
3418 * executing guest code so they'll trigger at the right time.
3419 */
3420 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3421 {
3422 if (!CPUMIsGuestDebugStateActive(pVCpu))
3423 {
3424 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3425 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3426 }
3427 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3428 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3429 }
3430 /*
3431 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3432 * must intercept #DB in order to maintain a correct DR6 guest value.
3433 */
3434 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3435 {
3436 fInterceptMovDRx = true;
3437 fInterceptDB = true;
3438 }
3439
3440 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3441 AssertRCReturn(rc, rc);
3442 }
3443
3444 /*
3445 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3446 */
3447 if (fInterceptDB)
3448 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3449 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3450 {
3451#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3452 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3453#endif
3454 }
3455 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3456 AssertRCReturn(rc, rc);
3457
3458 /*
3459 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3460 */
3461 if (fInterceptMovDRx)
3462 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3463 else
3464 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3465 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3466 AssertRCReturn(rc, rc);
3467
3468 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3469 return VINF_SUCCESS;
3470}
3471
3472
3473#ifdef VBOX_STRICT
3474/**
3475 * Strict function to validate segment registers.
3476 *
3477 * @remarks ASSUMES CR0 is up to date.
3478 */
3479static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3480{
3481 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3482 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3483 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3484 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3485 && ( !CPUMIsGuestInRealModeEx(pCtx)
3486 && !CPUMIsGuestInV86ModeEx(pCtx)))
3487 {
3488 /* Protected mode checks */
3489 /* CS */
3490 Assert(pCtx->cs.Attr.n.u1Present);
3491 Assert(!(pCtx->cs.Attr.u & 0xf00));
3492 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3493 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3494 || !(pCtx->cs.Attr.n.u1Granularity));
3495 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3496 || (pCtx->cs.Attr.n.u1Granularity));
3497 /* CS cannot be loaded with NULL in protected mode. */
3498 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3499 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3500 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3501 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3502 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3503 else
3504 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3505 /* SS */
3506 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3507 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3508 if ( !(pCtx->cr0 & X86_CR0_PE)
3509 || pCtx->cs.Attr.n.u4Type == 3)
3510 {
3511 Assert(!pCtx->ss.Attr.n.u2Dpl);
3512 }
3513 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3514 {
3515 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3516 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3517 Assert(pCtx->ss.Attr.n.u1Present);
3518 Assert(!(pCtx->ss.Attr.u & 0xf00));
3519 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3520 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3521 || !(pCtx->ss.Attr.n.u1Granularity));
3522 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3523 || (pCtx->ss.Attr.n.u1Granularity));
3524 }
3525 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3526 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3527 {
3528 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3529 Assert(pCtx->ds.Attr.n.u1Present);
3530 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3531 Assert(!(pCtx->ds.Attr.u & 0xf00));
3532 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3533 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3534 || !(pCtx->ds.Attr.n.u1Granularity));
3535 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3536 || (pCtx->ds.Attr.n.u1Granularity));
3537 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3538 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3539 }
3540 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3541 {
3542 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3543 Assert(pCtx->es.Attr.n.u1Present);
3544 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3545 Assert(!(pCtx->es.Attr.u & 0xf00));
3546 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3547 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3548 || !(pCtx->es.Attr.n.u1Granularity));
3549 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3550 || (pCtx->es.Attr.n.u1Granularity));
3551 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3552 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3553 }
3554 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3555 {
3556 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3557 Assert(pCtx->fs.Attr.n.u1Present);
3558 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3559 Assert(!(pCtx->fs.Attr.u & 0xf00));
3560 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3561 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3562 || !(pCtx->fs.Attr.n.u1Granularity));
3563 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3564 || (pCtx->fs.Attr.n.u1Granularity));
3565 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3566 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3567 }
3568 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3569 {
3570 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3571 Assert(pCtx->gs.Attr.n.u1Present);
3572 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3573 Assert(!(pCtx->gs.Attr.u & 0xf00));
3574 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3575 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3576 || !(pCtx->gs.Attr.n.u1Granularity));
3577 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3578 || (pCtx->gs.Attr.n.u1Granularity));
3579 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3580 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3581 }
3582 /* 64-bit capable CPUs. */
3583# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3584 Assert(!(pCtx->cs.u64Base >> 32));
3585 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3586 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3587 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3588# endif
3589 }
3590 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3591 || ( CPUMIsGuestInRealModeEx(pCtx)
3592 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3593 {
3594 /* Real and v86 mode checks. */
3595 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3596 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3597 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3598 {
3599 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3600 }
3601 else
3602 {
3603 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3604 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3605 }
3606
3607 /* CS */
3608 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3609 Assert(pCtx->cs.u32Limit == 0xffff);
3610 Assert(u32CSAttr == 0xf3);
3611 /* SS */
3612 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3613 Assert(pCtx->ss.u32Limit == 0xffff);
3614 Assert(u32SSAttr == 0xf3);
3615 /* DS */
3616 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3617 Assert(pCtx->ds.u32Limit == 0xffff);
3618 Assert(u32DSAttr == 0xf3);
3619 /* ES */
3620 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3621 Assert(pCtx->es.u32Limit == 0xffff);
3622 Assert(u32ESAttr == 0xf3);
3623 /* FS */
3624 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3625 Assert(pCtx->fs.u32Limit == 0xffff);
3626 Assert(u32FSAttr == 0xf3);
3627 /* GS */
3628 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3629 Assert(pCtx->gs.u32Limit == 0xffff);
3630 Assert(u32GSAttr == 0xf3);
3631 /* 64-bit capable CPUs. */
3632# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3633 Assert(!(pCtx->cs.u64Base >> 32));
3634 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3635 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3636 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3637# endif
3638 }
3639}
3640#endif /* VBOX_STRICT */
3641
3642
3643/**
3644 * Writes a guest segment register into the guest-state area in the VMCS.
3645 *
3646 * @returns VBox status code.
3647 * @param pVCpu Pointer to the VMCPU.
3648 * @param idxSel Index of the selector in the VMCS.
3649 * @param idxLimit Index of the segment limit in the VMCS.
3650 * @param idxBase Index of the segment base in the VMCS.
3651 * @param idxAccess Index of the access rights of the segment in the VMCS.
3652 * @param pSelReg Pointer to the segment selector.
3653 * @param pCtx Pointer to the guest-CPU context.
3654 *
3655 * @remarks No-long-jump zone!!!
3656 */
3657static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3658 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3659{
3660 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3661 AssertRCReturn(rc, rc);
3662 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3663 AssertRCReturn(rc, rc);
3664 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3665 AssertRCReturn(rc, rc);
3666
3667 uint32_t u32Access = pSelReg->Attr.u;
3668 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3669 {
3670 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3671 u32Access = 0xf3;
3672 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3673 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3674 }
3675 else
3676 {
3677 /*
3678 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3679 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3680 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3681 * loaded in protected-mode have their attribute as 0.
3682 */
3683 if (!u32Access)
3684 u32Access = X86DESCATTR_UNUSABLE;
3685 }
3686
3687 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3688 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3689 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3690
3691 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3692 AssertRCReturn(rc, rc);
3693 return rc;
3694}
3695
3696
3697/**
3698 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3699 * into the guest-state area in the VMCS.
3700 *
3701 * @returns VBox status code.
3702 * @param pVM Pointer to the VM.
3703 * @param pVCPU Pointer to the VMCPU.
3704 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3705 * out-of-sync. Make sure to update the required fields
3706 * before using them.
3707 *
3708 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
3709 * @remarks No-long-jump zone!!!
3710 */
3711static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3712{
3713 int rc = VERR_INTERNAL_ERROR_5;
3714 PVM pVM = pVCpu->CTX_SUFF(pVM);
3715
3716 /*
3717 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
3718 */
3719 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3720 {
3721 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3722 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3723 {
3724 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3725 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3726 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3727 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3728 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3729 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3730 }
3731
3732#ifdef VBOX_WITH_REM
3733 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3734 {
3735 Assert(pVM->hm.s.vmx.pRealModeTSS);
3736 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3737 if ( pVCpu->hm.s.vmx.fWasInRealMode
3738 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3739 {
3740 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3741 in real-mode (e.g. OpenBSD 4.0) */
3742 REMFlushTBs(pVM);
3743 Log4(("Load: Switch to protected mode detected!\n"));
3744 pVCpu->hm.s.vmx.fWasInRealMode = false;
3745 }
3746 }
3747#endif
3748 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3749 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3750 AssertRCReturn(rc, rc);
3751 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3752 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3753 AssertRCReturn(rc, rc);
3754 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3755 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3756 AssertRCReturn(rc, rc);
3757 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3758 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3759 AssertRCReturn(rc, rc);
3760 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3761 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3762 AssertRCReturn(rc, rc);
3763 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3764 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3765 AssertRCReturn(rc, rc);
3766
3767 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3768 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3769#ifdef VBOX_STRICT
3770 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3771#endif
3772 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3773 }
3774
3775 /*
3776 * Guest TR.
3777 */
3778 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3779 {
3780 /*
3781 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3782 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3783 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3784 */
3785 uint16_t u16Sel = 0;
3786 uint32_t u32Limit = 0;
3787 uint64_t u64Base = 0;
3788 uint32_t u32AccessRights = 0;
3789
3790 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3791 {
3792 u16Sel = pMixedCtx->tr.Sel;
3793 u32Limit = pMixedCtx->tr.u32Limit;
3794 u64Base = pMixedCtx->tr.u64Base;
3795 u32AccessRights = pMixedCtx->tr.Attr.u;
3796 }
3797 else
3798 {
3799 Assert(pVM->hm.s.vmx.pRealModeTSS);
3800 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3801
3802 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3803 RTGCPHYS GCPhys;
3804 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3805 AssertRCReturn(rc, rc);
3806
3807 X86DESCATTR DescAttr;
3808 DescAttr.u = 0;
3809 DescAttr.n.u1Present = 1;
3810 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3811
3812 u16Sel = 0;
3813 u32Limit = HM_VTX_TSS_SIZE;
3814 u64Base = GCPhys; /* in real-mode phys = virt. */
3815 u32AccessRights = DescAttr.u;
3816 }
3817
3818 /* Validate. */
3819 Assert(!(u16Sel & RT_BIT(2)));
3820 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3821 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3822 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3823 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3824 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3825 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3826 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3827 Assert( (u32Limit & 0xfff) == 0xfff
3828 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3829 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3830 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3831
3832 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3833 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3834 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3835 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3836
3837 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3838 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3839 }
3840
3841 /*
3842 * Guest GDTR.
3843 */
3844 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3845 {
3846 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3847 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3848
3849 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3850 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3851 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3852 }
3853
3854 /*
3855 * Guest LDTR.
3856 */
3857 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3858 {
3859 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3860 uint32_t u32Access = 0;
3861 if (!pMixedCtx->ldtr.Attr.u)
3862 u32Access = X86DESCATTR_UNUSABLE;
3863 else
3864 u32Access = pMixedCtx->ldtr.Attr.u;
3865
3866 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3867 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3868 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3869 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3870
3871 /* Validate. */
3872 if (!(u32Access & X86DESCATTR_UNUSABLE))
3873 {
3874 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3875 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3876 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3877 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3878 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3879 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3880 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3881 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3882 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3883 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3884 }
3885
3886 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3887 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3888 }
3889
3890 /*
3891 * Guest IDTR.
3892 */
3893 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3894 {
3895 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3896 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3897
3898 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3899 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3900 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3901 }
3902
3903 return VINF_SUCCESS;
3904}
3905
3906
3907/**
3908 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3909 * areas. These MSRs will automatically be loaded to the host CPU on every
3910 * successful VM entry and stored from the host CPU on every successful VM exit.
3911 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
3912 *
3913 * @returns VBox status code.
3914 * @param pVCpu Pointer to the VMCPU.
3915 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3916 * out-of-sync. Make sure to update the required fields
3917 * before using them.
3918 *
3919 * @remarks No-long-jump zone!!!
3920 */
3921static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3922{
3923 AssertPtr(pVCpu);
3924 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3925
3926 /*
3927 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3928 */
3929 int rc = VINF_SUCCESS;
3930 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3931 {
3932#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3933 PVM pVM = pVCpu->CTX_SUFF(pVM);
3934 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3935 uint32_t cGuestMsrs = 0;
3936
3937 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3938 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3939 * when the guest really is in 64-bit mode. */
3940 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3941 if (fSupportsLongMode)
3942 {
3943 pGuestMsr->u32Msr = MSR_K8_LSTAR;
3944 pGuestMsr->u32Reserved = 0;
3945 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3946 pGuestMsr++; cGuestMsrs++;
3947 pGuestMsr->u32Msr = MSR_K6_STAR;
3948 pGuestMsr->u32Reserved = 0;
3949 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3950 pGuestMsr++; cGuestMsrs++;
3951 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
3952 pGuestMsr->u32Reserved = 0;
3953 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3954 pGuestMsr++; cGuestMsrs++;
3955 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
3956 pGuestMsr->u32Reserved = 0;
3957 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3958 pGuestMsr++; cGuestMsrs++;
3959 }
3960
3961 /*
3962 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3963 * load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3964 */
3965 if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3966 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3967 {
3968 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
3969 pGuestMsr->u32Reserved = 0;
3970 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3971 AssertRCReturn(rc, rc);
3972 pGuestMsr++; cGuestMsrs++;
3973 }
3974
3975 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3976 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
3977 {
3978 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3979 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
3980 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3981 }
3982
3983 /* Update the VCPU's copy of the guest MSR count. */
3984 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3985 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3986 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3987#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3988
3989 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3990 }
3991
3992 /*
3993 * Guest Sysenter MSRs.
3994 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3995 * VM-exits on WRMSRs for these MSRs.
3996 */
3997 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3998 {
3999 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4000 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
4001 }
4002 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4003 {
4004 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4005 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
4006 }
4007 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4008 {
4009 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4010 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
4011 }
4012
4013 return rc;
4014}
4015
4016
4017/**
4018 * Loads the guest activity state into the guest-state area in the VMCS.
4019 *
4020 * @returns VBox status code.
4021 * @param pVCpu Pointer to the VMCPU.
4022 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4023 * out-of-sync. Make sure to update the required fields
4024 * before using them.
4025 *
4026 * @remarks No-long-jump zone!!!
4027 */
4028static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4029{
4030 /** @todo See if we can make use of other states, e.g.
4031 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4032 int rc = VINF_SUCCESS;
4033 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
4034 {
4035 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4036 AssertRCReturn(rc, rc);
4037 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
4038 }
4039 return rc;
4040}
4041
4042
4043/**
4044 * Sets up the appropriate function to run guest code.
4045 *
4046 * @returns VBox status code.
4047 * @param pVCpu Pointer to the VMCPU.
4048 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4049 * out-of-sync. Make sure to update the required fields
4050 * before using them.
4051 *
4052 * @remarks No-long-jump zone!!!
4053 */
4054static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4055{
4056 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4057 {
4058#ifndef VBOX_ENABLE_64_BITS_GUESTS
4059 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4060#endif
4061 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4062#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4063 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4064 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4065 {
4066 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4067 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS;
4068 }
4069#else
4070 /* 64-bit host or hybrid host. */
4071 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4072#endif
4073 }
4074 else
4075 {
4076 /* Guest is not in long mode, use the 32-bit handler. */
4077#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4078 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4079 {
4080 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4081 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS;
4082 }
4083#else
4084 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4085#endif
4086 }
4087 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4088 return VINF_SUCCESS;
4089}
4090
4091
4092/**
4093 * Wrapper for running the guest code in VT-x.
4094 *
4095 * @returns VBox strict status code.
4096 * @param pVM Pointer to the VM.
4097 * @param pVCpu Pointer to the VMCPU.
4098 * @param pCtx Pointer to the guest-CPU context.
4099 *
4100 * @remarks No-long-jump zone!!!
4101 */
4102DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4103{
4104 /*
4105 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4106 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4107 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4108 */
4109 const bool fResumeVM = !!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4110 /** @todo Add stats for resume vs launch. */
4111#ifdef VBOX_WITH_KERNEL_USING_XMM
4112 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4113#else
4114 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4115#endif
4116}
4117
4118
4119/**
4120 * Reports world-switch error and dumps some useful debug info.
4121 *
4122 * @param pVM Pointer to the VM.
4123 * @param pVCpu Pointer to the VMCPU.
4124 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4125 * @param pCtx Pointer to the guest-CPU context.
4126 * @param pVmxTransient Pointer to the VMX transient structure (only
4127 * exitReason updated).
4128 */
4129static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4130{
4131 Assert(pVM);
4132 Assert(pVCpu);
4133 Assert(pCtx);
4134 Assert(pVmxTransient);
4135 HMVMX_ASSERT_PREEMPT_SAFE();
4136
4137 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4138 switch (rcVMRun)
4139 {
4140 case VERR_VMX_INVALID_VMXON_PTR:
4141 AssertFailed();
4142 break;
4143 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4144 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4145 {
4146 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4147 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4148 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4149 AssertRC(rc);
4150
4151 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4152 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4153 Cannot do it here as we may have been long preempted. */
4154
4155#ifdef VBOX_STRICT
4156 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4157 pVmxTransient->uExitReason));
4158 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4159 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4160 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4161 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4162 else
4163 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4164 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4165 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4166
4167 /* VMX control bits. */
4168 uint32_t u32Val;
4169 uint64_t u64Val;
4170 HMVMXHCUINTREG uHCReg;
4171 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4172 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4173 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4174 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4175 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4176 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4177 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4178 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4179 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4180 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4181 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4182 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4183 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4184 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4185 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4186 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4187 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4188 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4189 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4190 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4191 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4192 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4193 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4194 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4195 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4196 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4197 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4198 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4199 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4200 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4201 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4202 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4203 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4204 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4205 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4206 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4207 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4208 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4209 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4210 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4211 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4212 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4213
4214 /* Guest bits. */
4215 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4216 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4217 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4218 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4219 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4220 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4221 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4222 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4223
4224 /* Host bits. */
4225 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4226 Log4(("Host CR0 %#RHr\n", uHCReg));
4227 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4228 Log4(("Host CR3 %#RHr\n", uHCReg));
4229 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4230 Log4(("Host CR4 %#RHr\n", uHCReg));
4231
4232 RTGDTR HostGdtr;
4233 PCX86DESCHC pDesc;
4234 ASMGetGDTR(&HostGdtr);
4235 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4236 Log4(("Host CS %#08x\n", u32Val));
4237 if (u32Val < HostGdtr.cbGdt)
4238 {
4239 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4240 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4241 }
4242
4243 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4244 Log4(("Host DS %#08x\n", u32Val));
4245 if (u32Val < HostGdtr.cbGdt)
4246 {
4247 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4248 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4249 }
4250
4251 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4252 Log4(("Host ES %#08x\n", u32Val));
4253 if (u32Val < HostGdtr.cbGdt)
4254 {
4255 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4256 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4257 }
4258
4259 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4260 Log4(("Host FS %#08x\n", u32Val));
4261 if (u32Val < HostGdtr.cbGdt)
4262 {
4263 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4264 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4265 }
4266
4267 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4268 Log4(("Host GS %#08x\n", u32Val));
4269 if (u32Val < HostGdtr.cbGdt)
4270 {
4271 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4272 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4273 }
4274
4275 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4276 Log4(("Host SS %#08x\n", u32Val));
4277 if (u32Val < HostGdtr.cbGdt)
4278 {
4279 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4280 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4281 }
4282
4283 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4284 Log4(("Host TR %#08x\n", u32Val));
4285 if (u32Val < HostGdtr.cbGdt)
4286 {
4287 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4288 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4289 }
4290
4291 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4292 Log4(("Host TR Base %#RHv\n", uHCReg));
4293 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4294 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4295 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4296 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4297 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4298 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4299 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4300 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4301 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4302 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4303 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4304 Log4(("Host RSP %#RHv\n", uHCReg));
4305 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4306 Log4(("Host RIP %#RHv\n", uHCReg));
4307# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4308 if (HMVMX_IS_64BIT_HOST_MODE())
4309 {
4310 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4311 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4312 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4313 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4314 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4315 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4316 }
4317# endif
4318#endif /* VBOX_STRICT */
4319 break;
4320 }
4321
4322 default:
4323 /* Impossible */
4324 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4325 break;
4326 }
4327 NOREF(pVM);
4328}
4329
4330
4331#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4332#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4333# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4334#endif
4335#ifdef VBOX_STRICT
4336static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4337{
4338 switch (idxField)
4339 {
4340 case VMX_VMCS_GUEST_RIP:
4341 case VMX_VMCS_GUEST_RSP:
4342 case VMX_VMCS_GUEST_SYSENTER_EIP:
4343 case VMX_VMCS_GUEST_SYSENTER_ESP:
4344 case VMX_VMCS_GUEST_GDTR_BASE:
4345 case VMX_VMCS_GUEST_IDTR_BASE:
4346 case VMX_VMCS_GUEST_CS_BASE:
4347 case VMX_VMCS_GUEST_DS_BASE:
4348 case VMX_VMCS_GUEST_ES_BASE:
4349 case VMX_VMCS_GUEST_FS_BASE:
4350 case VMX_VMCS_GUEST_GS_BASE:
4351 case VMX_VMCS_GUEST_SS_BASE:
4352 case VMX_VMCS_GUEST_LDTR_BASE:
4353 case VMX_VMCS_GUEST_TR_BASE:
4354 case VMX_VMCS_GUEST_CR3:
4355 return true;
4356 }
4357 return false;
4358}
4359
4360static bool hmR0VmxIsValidReadField(uint32_t idxField)
4361{
4362 switch (idxField)
4363 {
4364 /* Read-only fields. */
4365 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4366 return true;
4367 }
4368 /* Remaining readable fields should also be writable. */
4369 return hmR0VmxIsValidWriteField(idxField);
4370}
4371#endif /* VBOX_STRICT */
4372
4373
4374/**
4375 * Executes the specified handler in 64-bit mode.
4376 *
4377 * @returns VBox status code.
4378 * @param pVM Pointer to the VM.
4379 * @param pVCpu Pointer to the VMCPU.
4380 * @param pCtx Pointer to the guest CPU context.
4381 * @param enmOp The operation to perform.
4382 * @param cbParam Number of parameters.
4383 * @param paParam Array of 32-bit parameters.
4384 */
4385VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4386 uint32_t *paParam)
4387{
4388 int rc, rc2;
4389 PHMGLOBALCPUINFO pCpu;
4390 RTHCPHYS HCPhysCpuPage;
4391 RTCCUINTREG uOldEflags;
4392
4393 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4394 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4395 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4396 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4397
4398#ifdef VBOX_STRICT
4399 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4400 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4401
4402 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4403 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4404#endif
4405
4406 /* Disable interrupts. */
4407 uOldEflags = ASMIntDisableFlags();
4408
4409#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4410 RTCPUID idHostCpu = RTMpCpuId();
4411 CPUMR0SetLApic(pVCpu, idHostCpu);
4412#endif
4413
4414 pCpu = HMR0GetCurrentCpu();
4415 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4416
4417 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4418 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4419
4420 /* Leave VMX Root Mode. */
4421 VMXDisable();
4422
4423 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4424
4425 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4426 CPUMSetHyperEIP(pVCpu, enmOp);
4427 for (int i = (int)cbParam - 1; i >= 0; i--)
4428 CPUMPushHyper(pVCpu, paParam[i]);
4429
4430 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4431
4432 /* Call the switcher. */
4433 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4434 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4435
4436 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4437 /* Make sure the VMX instructions don't cause #UD faults. */
4438 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4439
4440 /* Re-enter VMX Root Mode */
4441 rc2 = VMXEnable(HCPhysCpuPage);
4442 if (RT_FAILURE(rc2))
4443 {
4444 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4445 ASMSetFlags(uOldEflags);
4446 return rc2;
4447 }
4448
4449 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4450 AssertRC(rc2);
4451 Assert(!(ASMGetFlags() & X86_EFL_IF));
4452 ASMSetFlags(uOldEflags);
4453 return rc;
4454}
4455
4456
4457/**
4458 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4459 * supporting 64-bit guests.
4460 *
4461 * @returns VBox status code.
4462 * @param fResume Whether to VMLAUNCH or VMRESUME.
4463 * @param pCtx Pointer to the guest-CPU context.
4464 * @param pCache Pointer to the VMCS cache.
4465 * @param pVM Pointer to the VM.
4466 * @param pVCpu Pointer to the VMCPU.
4467 */
4468DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4469{
4470 uint32_t aParam[6];
4471 PHMGLOBALCPUINFO pCpu = NULL;
4472 RTHCPHYS HCPhysCpuPage = 0;
4473 int rc = VERR_INTERNAL_ERROR_5;
4474
4475 pCpu = HMR0GetCurrentCpu();
4476 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4477
4478#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4479 pCache->uPos = 1;
4480 pCache->interPD = PGMGetInterPaeCR3(pVM);
4481 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4482#endif
4483
4484#ifdef VBOX_STRICT
4485 pCache->TestIn.HCPhysCpuPage = 0;
4486 pCache->TestIn.HCPhysVmcs = 0;
4487 pCache->TestIn.pCache = 0;
4488 pCache->TestOut.HCPhysVmcs = 0;
4489 pCache->TestOut.pCache = 0;
4490 pCache->TestOut.pCtx = 0;
4491 pCache->TestOut.eflags = 0;
4492#endif
4493
4494 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4495 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4496 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4497 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4498 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4499 aParam[5] = 0;
4500
4501#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4502 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4503 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4504#endif
4505 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4506
4507#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4508 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4509 Assert(pCtx->dr[4] == 10);
4510 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4511#endif
4512
4513#ifdef VBOX_STRICT
4514 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4515 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4516 pVCpu->hm.s.vmx.HCPhysVmcs));
4517 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4518 pCache->TestOut.HCPhysVmcs));
4519 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4520 pCache->TestOut.pCache));
4521 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4522 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4523 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4524 pCache->TestOut.pCtx));
4525 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4526#endif
4527 return rc;
4528}
4529
4530
4531/**
4532 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4533 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4534 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4535 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4536 *
4537 * @returns VBox status code.
4538 * @param pVM Pointer to the VM.
4539 * @param pVCpu Pointer to the VMCPU.
4540 */
4541static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4542{
4543#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4544{ \
4545 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4546 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4547 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4548 ++cReadFields; \
4549}
4550
4551 AssertPtr(pVM);
4552 AssertPtr(pVCpu);
4553 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4554 uint32_t cReadFields = 0;
4555
4556 /*
4557 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4558 * and serve to indicate exceptions to the rules.
4559 */
4560
4561 /* Guest-natural selector base fields. */
4562#if 0
4563 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4564 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4565 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4566#endif
4567 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4568 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4569 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4570 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4571 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4572 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4573 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4574 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4575 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4576 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4577 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4578 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4579#if 0
4580 /* Unused natural width guest-state fields. */
4581 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4582 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4583#endif
4584 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4585 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4586
4587 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4588#if 0
4589 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4590 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4591 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4592 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4593 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4594 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4595 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4596 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4597 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4598#endif
4599
4600 /* Natural width guest-state fields. */
4601 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4602#if 0
4603 /* Currently unused field. */
4604 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4605#endif
4606
4607 if (pVM->hm.s.fNestedPaging)
4608 {
4609 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4610 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4611 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4612 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4613 }
4614 else
4615 {
4616 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4617 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4618 }
4619
4620#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4621 return VINF_SUCCESS;
4622}
4623
4624
4625/**
4626 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4627 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4628 * darwin, running 64-bit guests).
4629 *
4630 * @returns VBox status code.
4631 * @param pVCpu Pointer to the VMCPU.
4632 * @param idxField The VMCS field encoding.
4633 * @param u64Val 16, 32 or 64 bits value.
4634 */
4635VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4636{
4637 int rc;
4638 switch (idxField)
4639 {
4640 /*
4641 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4642 */
4643 /* 64-bit Control fields. */
4644 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4645 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4646 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4647 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4648 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4649 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4650 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4651 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4652 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4653 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4654 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4655 case VMX_VMCS64_CTRL_EPTP_FULL:
4656 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4657 /* 64-bit Guest-state fields. */
4658 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4659 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4660 case VMX_VMCS64_GUEST_PAT_FULL:
4661 case VMX_VMCS64_GUEST_EFER_FULL:
4662 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4663 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4664 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4665 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4666 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4667 /* 64-bit Host-state fields. */
4668 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4669 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4670 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4671 {
4672 rc = VMXWriteVmcs32(idxField, u64Val);
4673 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4674 break;
4675 }
4676
4677 /*
4678 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4679 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4680 */
4681 /* Natural-width Guest-state fields. */
4682 case VMX_VMCS_GUEST_CR3:
4683 case VMX_VMCS_GUEST_ES_BASE:
4684 case VMX_VMCS_GUEST_CS_BASE:
4685 case VMX_VMCS_GUEST_SS_BASE:
4686 case VMX_VMCS_GUEST_DS_BASE:
4687 case VMX_VMCS_GUEST_FS_BASE:
4688 case VMX_VMCS_GUEST_GS_BASE:
4689 case VMX_VMCS_GUEST_LDTR_BASE:
4690 case VMX_VMCS_GUEST_TR_BASE:
4691 case VMX_VMCS_GUEST_GDTR_BASE:
4692 case VMX_VMCS_GUEST_IDTR_BASE:
4693 case VMX_VMCS_GUEST_RSP:
4694 case VMX_VMCS_GUEST_RIP:
4695 case VMX_VMCS_GUEST_SYSENTER_ESP:
4696 case VMX_VMCS_GUEST_SYSENTER_EIP:
4697 {
4698 if (!(u64Val >> 32))
4699 {
4700 /* If this field is 64-bit, VT-x will zero out the top bits. */
4701 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4702 }
4703 else
4704 {
4705 /* Assert that only the 32->64 switcher case should ever come here. */
4706 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4707 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4708 }
4709 break;
4710 }
4711
4712 default:
4713 {
4714 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4715 rc = VERR_INVALID_PARAMETER;
4716 break;
4717 }
4718 }
4719 AssertRCReturn(rc, rc);
4720 return rc;
4721}
4722
4723
4724/**
4725 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4726 * hosts (except darwin) for 64-bit guests.
4727 *
4728 * @param pVCpu Pointer to the VMCPU.
4729 * @param idxField The VMCS field encoding.
4730 * @param u64Val 16, 32 or 64 bits value.
4731 */
4732VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4733{
4734 AssertPtr(pVCpu);
4735 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4736
4737 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4738 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4739
4740 /* Make sure there are no duplicates. */
4741 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4742 {
4743 if (pCache->Write.aField[i] == idxField)
4744 {
4745 pCache->Write.aFieldVal[i] = u64Val;
4746 return VINF_SUCCESS;
4747 }
4748 }
4749
4750 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4751 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4752 pCache->Write.cValidEntries++;
4753 return VINF_SUCCESS;
4754}
4755
4756/* Enable later when the assembly code uses these as callbacks. */
4757#if 0
4758/*
4759 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4760 *
4761 * @param pVCpu Pointer to the VMCPU.
4762 * @param pCache Pointer to the VMCS cache.
4763 *
4764 * @remarks No-long-jump zone!!!
4765 */
4766VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4767{
4768 AssertPtr(pCache);
4769 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4770 {
4771 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4772 AssertRC(rc);
4773 }
4774 pCache->Write.cValidEntries = 0;
4775}
4776
4777
4778/**
4779 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
4780 *
4781 * @param pVCpu Pointer to the VMCPU.
4782 * @param pCache Pointer to the VMCS cache.
4783 *
4784 * @remarks No-long-jump zone!!!
4785 */
4786VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4787{
4788 AssertPtr(pCache);
4789 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4790 {
4791 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4792 AssertRC(rc);
4793 }
4794}
4795#endif
4796#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4797
4798
4799/**
4800 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4801 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4802 * timer.
4803 *
4804 * @returns VBox status code.
4805 * @param pVCpu Pointer to the VMCPU.
4806 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4807 * out-of-sync. Make sure to update the required fields
4808 * before using them.
4809 * @remarks No-long-jump zone!!!
4810 */
4811static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4812{
4813 int rc = VERR_INTERNAL_ERROR_5;
4814 bool fOffsettedTsc = false;
4815 PVM pVM = pVCpu->CTX_SUFF(pVM);
4816 if (pVM->hm.s.vmx.fUsePreemptTimer)
4817 {
4818 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4819
4820 /* Make sure the returned values have sane upper and lower boundaries. */
4821 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4822 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4823 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4824 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4825
4826 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4827 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4828 }
4829 else
4830 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4831
4832 if (fOffsettedTsc)
4833 {
4834 uint64_t u64CurTSC = ASMReadTSC();
4835 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4836 {
4837 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4838 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4839
4840 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4841 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4842 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4843 }
4844 else
4845 {
4846 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4847 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4848 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4849 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4850 }
4851 }
4852 else
4853 {
4854 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4855 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4856 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4857 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4858 }
4859}
4860
4861
4862/**
4863 * Determines if an exception is a contributory exception. Contributory
4864 * exceptions are ones which can cause double-faults. Page-fault is
4865 * intentionally not included here as it's a conditional contributory exception.
4866 *
4867 * @returns true if the exception is contributory, false otherwise.
4868 * @param uVector The exception vector.
4869 */
4870DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4871{
4872 switch (uVector)
4873 {
4874 case X86_XCPT_GP:
4875 case X86_XCPT_SS:
4876 case X86_XCPT_NP:
4877 case X86_XCPT_TS:
4878 case X86_XCPT_DE:
4879 return true;
4880 default:
4881 break;
4882 }
4883 return false;
4884}
4885
4886
4887/**
4888 * Sets an event as a pending event to be injected into the guest.
4889 *
4890 * @param pVCpu Pointer to the VMCPU.
4891 * @param u32IntrInfo The VM-entry interruption-information field.
4892 * @param cbInstr The VM-entry instruction length in bytes (for software
4893 * interrupts, exceptions and privileged software
4894 * exceptions).
4895 * @param u32ErrCode The VM-entry exception error code.
4896 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4897 * page-fault.
4898 *
4899 * @remarks Statistics counter assumes this is a guest event being injected or
4900 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4901 * always incremented.
4902 */
4903DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4904 RTGCUINTPTR GCPtrFaultAddress)
4905{
4906 Assert(!pVCpu->hm.s.Event.fPending);
4907 pVCpu->hm.s.Event.fPending = true;
4908 pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4909 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4910 pVCpu->hm.s.Event.cbInstr = cbInstr;
4911 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4912
4913 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4914}
4915
4916
4917/**
4918 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4919 *
4920 * @param pVCpu Pointer to the VMCPU.
4921 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4922 * out-of-sync. Make sure to update the required fields
4923 * before using them.
4924 */
4925DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4926{
4927 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
4928 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4929 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4930 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
4931}
4932
4933
4934/**
4935 * Handle a condition that occurred while delivering an event through the guest
4936 * IDT.
4937 *
4938 * @returns VBox status code (informational error codes included).
4939 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
4940 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4941 * continue execution of the guest which will delivery the #DF.
4942 * @retval VINF_EM_RESET if we detected a triple-fault condition.
4943 *
4944 * @param pVCpu Pointer to the VMCPU.
4945 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4946 * out-of-sync. Make sure to update the required fields
4947 * before using them.
4948 * @param pVmxTransient Pointer to the VMX transient structure.
4949 *
4950 * @remarks No-long-jump zone!!!
4951 */
4952static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4953{
4954 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4955 AssertRCReturn(rc, rc);
4956 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4957 {
4958 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4959 AssertRCReturn(rc, rc);
4960
4961 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4962 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4963 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4964
4965 typedef enum
4966 {
4967 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4968 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4969 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4970 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4971 } VMXREFLECTXCPT;
4972
4973 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4974 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4975 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4976 {
4977 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4978 {
4979 enmReflect = VMXREFLECTXCPT_XCPT;
4980#ifdef VBOX_STRICT
4981 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4982 && uExitVector == X86_XCPT_PF)
4983 {
4984 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4985 }
4986#endif
4987 if ( uExitVector == X86_XCPT_PF
4988 && uIdtVector == X86_XCPT_PF)
4989 {
4990 pVmxTransient->fVectoringPF = true;
4991 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4992 }
4993 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4994 && hmR0VmxIsContributoryXcpt(uExitVector)
4995 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
4996 || uIdtVector == X86_XCPT_PF))
4997 {
4998 enmReflect = VMXREFLECTXCPT_DF;
4999 }
5000 else if (uIdtVector == X86_XCPT_DF)
5001 enmReflect = VMXREFLECTXCPT_TF;
5002 }
5003 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5004 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5005 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5006 {
5007 /*
5008 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5009 * (whatever they are) as they reoccur when restarting the instruction.
5010 */
5011 enmReflect = VMXREFLECTXCPT_XCPT;
5012 }
5013 }
5014 else
5015 {
5016 /*
5017 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5018 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5019 * original exception to the guest after handling the VM-exit.
5020 */
5021 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5022 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5023 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5024 {
5025 enmReflect = VMXREFLECTXCPT_XCPT;
5026 }
5027 }
5028
5029 switch (enmReflect)
5030 {
5031 case VMXREFLECTXCPT_XCPT:
5032 {
5033 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5034 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5035 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5036
5037 uint32_t u32ErrCode = 0;
5038 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5039 {
5040 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5041 AssertRCReturn(rc, rc);
5042 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5043 }
5044
5045 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5046 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5047 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5048 rc = VINF_SUCCESS;
5049 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5050 pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
5051
5052 break;
5053 }
5054
5055 case VMXREFLECTXCPT_DF:
5056 {
5057 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5058 rc = VINF_HM_DOUBLE_FAULT;
5059 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5060 pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
5061
5062 break;
5063 }
5064
5065 case VMXREFLECTXCPT_TF:
5066 {
5067 rc = VINF_EM_RESET;
5068 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5069 uExitVector));
5070 break;
5071 }
5072
5073 default:
5074 Assert(rc == VINF_SUCCESS);
5075 break;
5076 }
5077 }
5078 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5079 return rc;
5080}
5081
5082
5083/**
5084 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5085 *
5086 * @returns VBox status code.
5087 * @param pVCpu Pointer to the VMCPU.
5088 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5089 * out-of-sync. Make sure to update the required fields
5090 * before using them.
5091 *
5092 * @remarks No-long-jump zone!!!
5093 */
5094static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5095{
5096 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5097 {
5098 uint32_t uVal = 0;
5099 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5100 AssertRCReturn(rc, rc);
5101 uint32_t uShadow = 0;
5102 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5103 AssertRCReturn(rc, rc);
5104
5105 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5106 CPUMSetGuestCR0(pVCpu, uVal);
5107 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5108 }
5109 return VINF_SUCCESS;
5110}
5111
5112
5113/**
5114 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5115 *
5116 * @returns VBox status code.
5117 * @param pVCpu Pointer to the VMCPU.
5118 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5119 * out-of-sync. Make sure to update the required fields
5120 * before using them.
5121 *
5122 * @remarks No-long-jump zone!!!
5123 */
5124static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5125{
5126 int rc = VINF_SUCCESS;
5127 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5128 {
5129 uint32_t uVal = 0;
5130 uint32_t uShadow = 0;
5131 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5132 AssertRCReturn(rc, rc);
5133 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5134 AssertRCReturn(rc, rc);
5135
5136 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5137 CPUMSetGuestCR4(pVCpu, uVal);
5138 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5139 }
5140 return rc;
5141}
5142
5143
5144/**
5145 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5146 *
5147 * @returns VBox status code.
5148 * @param pVCpu Pointer to the VMCPU.
5149 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5150 * out-of-sync. Make sure to update the required fields
5151 * before using them.
5152 *
5153 * @remarks No-long-jump zone!!!
5154 */
5155static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5156{
5157 int rc = VINF_SUCCESS;
5158 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5159 {
5160 uint64_t u64Val = 0;
5161 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5162 AssertRCReturn(rc, rc);
5163
5164 pMixedCtx->rip = u64Val;
5165 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5166 }
5167 return rc;
5168}
5169
5170
5171/**
5172 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5173 *
5174 * @returns VBox status code.
5175 * @param pVCpu Pointer to the VMCPU.
5176 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5177 * out-of-sync. Make sure to update the required fields
5178 * before using them.
5179 *
5180 * @remarks No-long-jump zone!!!
5181 */
5182static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5183{
5184 int rc = VINF_SUCCESS;
5185 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5186 {
5187 uint64_t u64Val = 0;
5188 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5189 AssertRCReturn(rc, rc);
5190
5191 pMixedCtx->rsp = u64Val;
5192 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5193 }
5194 return rc;
5195}
5196
5197
5198/**
5199 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5200 *
5201 * @returns VBox status code.
5202 * @param pVCpu Pointer to the VMCPU.
5203 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5204 * out-of-sync. Make sure to update the required fields
5205 * before using them.
5206 *
5207 * @remarks No-long-jump zone!!!
5208 */
5209static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5210{
5211 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5212 {
5213 uint32_t uVal = 0;
5214 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5215 AssertRCReturn(rc, rc);
5216
5217 pMixedCtx->eflags.u32 = uVal;
5218 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5219 {
5220 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5221 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5222
5223 pMixedCtx->eflags.Bits.u1VM = 0;
5224 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5225 }
5226
5227 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5228 }
5229 return VINF_SUCCESS;
5230}
5231
5232
5233/**
5234 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5235 * guest-CPU context.
5236 */
5237DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5238{
5239 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5240 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5241 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5242 return rc;
5243}
5244
5245
5246/**
5247 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5248 * from the guest-state area in the VMCS.
5249 *
5250 * @param pVCpu Pointer to the VMCPU.
5251 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5252 * out-of-sync. Make sure to update the required fields
5253 * before using them.
5254 *
5255 * @remarks No-long-jump zone!!!
5256 */
5257static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5258{
5259 uint32_t uIntrState = 0;
5260 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5261 AssertRC(rc);
5262
5263 if (!uIntrState)
5264 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5265 else
5266 {
5267 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5268 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5269 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5270 AssertRC(rc);
5271 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5272 AssertRC(rc);
5273
5274 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5275 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5276 }
5277}
5278
5279
5280/**
5281 * Saves the guest's activity state.
5282 *
5283 * @returns VBox status code.
5284 * @param pVCpu Pointer to the VMCPU.
5285 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5286 * out-of-sync. Make sure to update the required fields
5287 * before using them.
5288 *
5289 * @remarks No-long-jump zone!!!
5290 */
5291static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5292{
5293 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5294 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5295 return VINF_SUCCESS;
5296}
5297
5298
5299/**
5300 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5301 * the current VMCS into the guest-CPU context.
5302 *
5303 * @returns VBox status code.
5304 * @param pVCpu Pointer to the VMCPU.
5305 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5306 * out-of-sync. Make sure to update the required fields
5307 * before using them.
5308 *
5309 * @remarks No-long-jump zone!!!
5310 */
5311static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5312{
5313 int rc = VINF_SUCCESS;
5314 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5315 {
5316 uint32_t u32Val = 0;
5317 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5318 pMixedCtx->SysEnter.cs = u32Val;
5319 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5320 }
5321
5322 uint64_t u64Val = 0;
5323 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5324 {
5325 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5326 pMixedCtx->SysEnter.eip = u64Val;
5327 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5328 }
5329 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5330 {
5331 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5332 pMixedCtx->SysEnter.esp = u64Val;
5333 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5334 }
5335 return rc;
5336}
5337
5338
5339/**
5340 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5341 * context.
5342 *
5343 * @returns VBox status code.
5344 * @param pVCpu Pointer to the VMCPU.
5345 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5346 * out-of-sync. Make sure to update the required fields
5347 * before using them.
5348 *
5349 * @remarks No-long-jump zone!!!
5350 */
5351static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5352{
5353 int rc = VINF_SUCCESS;
5354 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5355 {
5356 uint64_t u64Val = 0;
5357 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5358 pMixedCtx->fs.u64Base = u64Val;
5359 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5360 }
5361 return rc;
5362}
5363
5364
5365/**
5366 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5367 * context.
5368 *
5369 * @returns VBox status code.
5370 * @param pVCpu Pointer to the VMCPU.
5371 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5372 * out-of-sync. Make sure to update the required fields
5373 * before using them.
5374 *
5375 * @remarks No-long-jump zone!!!
5376 */
5377static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5378{
5379 int rc = VINF_SUCCESS;
5380 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5381 {
5382 uint64_t u64Val = 0;
5383 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5384 pMixedCtx->gs.u64Base = u64Val;
5385 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5386 }
5387 return rc;
5388}
5389
5390
5391/**
5392 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5393 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5394 * and TSC_AUX.
5395 *
5396 * @returns VBox status code.
5397 * @param pVCpu Pointer to the VMCPU.
5398 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5399 * out-of-sync. Make sure to update the required fields
5400 * before using them.
5401 *
5402 * @remarks No-long-jump zone!!!
5403 */
5404static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5405{
5406 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5407 return VINF_SUCCESS;
5408
5409#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5410 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5411 {
5412 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5413 pMsr += i;
5414 switch (pMsr->u32Msr)
5415 {
5416 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5417 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5418 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5419 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5420 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5421 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5422 default:
5423 {
5424 AssertFailed();
5425 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5426 }
5427 }
5428 }
5429#endif
5430
5431 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5432 return VINF_SUCCESS;
5433}
5434
5435
5436/**
5437 * Saves the guest control registers from the current VMCS into the guest-CPU
5438 * context.
5439 *
5440 * @returns VBox status code.
5441 * @param pVCpu Pointer to the VMCPU.
5442 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5443 * out-of-sync. Make sure to update the required fields
5444 * before using them.
5445 *
5446 * @remarks No-long-jump zone!!!
5447 */
5448static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5449{
5450 /* Guest CR0. Guest FPU. */
5451 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5452 AssertRCReturn(rc, rc);
5453
5454 /* Guest CR4. */
5455 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5456 AssertRCReturn(rc, rc);
5457
5458 /* Guest CR2 - updated always during the world-switch or in #PF. */
5459 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5460 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5461 {
5462 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5463 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5464
5465 PVM pVM = pVCpu->CTX_SUFF(pVM);
5466 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5467 || ( pVM->hm.s.fNestedPaging
5468 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5469 {
5470 uint64_t u64Val = 0;
5471 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5472 if (pMixedCtx->cr3 != u64Val)
5473 {
5474 CPUMSetGuestCR3(pVCpu, u64Val);
5475 if (VMMRZCallRing3IsEnabled(pVCpu))
5476 {
5477 PGMUpdateCR3(pVCpu, u64Val);
5478 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5479 }
5480 else
5481 {
5482 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5483 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5484 }
5485 }
5486
5487 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5488 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5489 {
5490 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5491 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5492 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5493 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5494
5495 if (VMMRZCallRing3IsEnabled(pVCpu))
5496 {
5497 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5498 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5499 }
5500 else
5501 {
5502 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5503 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5504 }
5505 }
5506 }
5507
5508 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5509 }
5510
5511 /*
5512 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5513 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5514 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5515 *
5516 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5517 */
5518 if (VMMRZCallRing3IsEnabled(pVCpu))
5519 {
5520 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5521 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5522
5523 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5524 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5525
5526 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5527 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5528 }
5529
5530 return rc;
5531}
5532
5533
5534/**
5535 * Reads a guest segment register from the current VMCS into the guest-CPU
5536 * context.
5537 *
5538 * @returns VBox status code.
5539 * @param pVCpu Pointer to the VMCPU.
5540 * @param idxSel Index of the selector in the VMCS.
5541 * @param idxLimit Index of the segment limit in the VMCS.
5542 * @param idxBase Index of the segment base in the VMCS.
5543 * @param idxAccess Index of the access rights of the segment in the VMCS.
5544 * @param pSelReg Pointer to the segment selector.
5545 *
5546 * @remarks No-long-jump zone!!!
5547 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5548 * macro as that takes care of whether to read from the VMCS cache or
5549 * not.
5550 */
5551DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5552 PCPUMSELREG pSelReg)
5553{
5554 uint32_t u32Val = 0;
5555 int rc = VMXReadVmcs32(idxSel, &u32Val);
5556 AssertRCReturn(rc, rc);
5557 pSelReg->Sel = (uint16_t)u32Val;
5558 pSelReg->ValidSel = (uint16_t)u32Val;
5559 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5560
5561 rc = VMXReadVmcs32(idxLimit, &u32Val);
5562 AssertRCReturn(rc, rc);
5563 pSelReg->u32Limit = u32Val;
5564
5565 uint64_t u64Val = 0;
5566 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5567 AssertRCReturn(rc, rc);
5568 pSelReg->u64Base = u64Val;
5569
5570 rc = VMXReadVmcs32(idxAccess, &u32Val);
5571 AssertRCReturn(rc, rc);
5572 pSelReg->Attr.u = u32Val;
5573
5574 /*
5575 * If VT-x marks the segment as unusable, most other bits remain undefined:
5576 * - For CS the L, D and G bits have meaning.
5577 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5578 * - For the remaining data segments no bits are defined.
5579 *
5580 * The present bit and the unusable bit has been observed to be set at the
5581 * same time (the selector was supposed to invalid as we started executing
5582 * a V8086 interrupt in ring-0).
5583 *
5584 * What should be important for the rest of the VBox code that the P bit is
5585 * cleared. Some of the other VBox code recognizes the unusable bit, but
5586 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5587 * safe side here, we'll strip off P and other bits we don't care about. If
5588 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5589 *
5590 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5591 */
5592 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5593 {
5594 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5595
5596 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5597 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5598 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5599
5600 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5601#ifdef DEBUG_bird
5602 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5603 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5604 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5605#endif
5606 }
5607 return VINF_SUCCESS;
5608}
5609
5610
5611#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5612# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5613 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5614 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5615#else
5616# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5617 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5618 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5619#endif
5620
5621
5622/**
5623 * Saves the guest segment registers from the current VMCS into the guest-CPU
5624 * context.
5625 *
5626 * @returns VBox status code.
5627 * @param pVCpu Pointer to the VMCPU.
5628 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5629 * out-of-sync. Make sure to update the required fields
5630 * before using them.
5631 *
5632 * @remarks No-long-jump zone!!!
5633 */
5634static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5635{
5636 /* Guest segment registers. */
5637 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5638 {
5639 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5640 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5641 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5642 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5643 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5644 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5645 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5646
5647 /* Restore segment attributes for real-on-v86 mode hack. */
5648 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5649 {
5650 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5651 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5652 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5653 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5654 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5655 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5656 }
5657 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5658 }
5659
5660 return VINF_SUCCESS;
5661}
5662
5663
5664/**
5665 * Saves the guest descriptor table registers and task register from the current
5666 * VMCS into the guest-CPU context.
5667 *
5668 * @returns VBox status code.
5669 * @param pVCpu Pointer to the VMCPU.
5670 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5671 * out-of-sync. Make sure to update the required fields
5672 * before using them.
5673 *
5674 * @remarks No-long-jump zone!!!
5675 */
5676static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5677{
5678 int rc = VINF_SUCCESS;
5679
5680 /* Guest LDTR. */
5681 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5682 {
5683 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5684 AssertRCReturn(rc, rc);
5685 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5686 }
5687
5688 /* Guest GDTR. */
5689 uint64_t u64Val = 0;
5690 uint32_t u32Val = 0;
5691 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5692 {
5693 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5694 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5695 pMixedCtx->gdtr.pGdt = u64Val;
5696 pMixedCtx->gdtr.cbGdt = u32Val;
5697 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
5698 }
5699
5700 /* Guest IDTR. */
5701 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5702 {
5703 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5704 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5705 pMixedCtx->idtr.pIdt = u64Val;
5706 pMixedCtx->idtr.cbIdt = u32Val;
5707 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
5708 }
5709
5710 /* Guest TR. */
5711 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5712 {
5713 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5714 AssertRCReturn(rc, rc);
5715
5716 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5717 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5718 {
5719 rc = VMXLOCAL_READ_SEG(TR, tr);
5720 AssertRCReturn(rc, rc);
5721 }
5722 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
5723 }
5724 return rc;
5725}
5726
5727#undef VMXLOCAL_READ_SEG
5728
5729
5730/**
5731 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5732 * context.
5733 *
5734 * @returns VBox status code.
5735 * @param pVCpu Pointer to the VMCPU.
5736 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5737 * out-of-sync. Make sure to update the required fields
5738 * before using them.
5739 *
5740 * @remarks No-long-jump zone!!!
5741 */
5742static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5743{
5744 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5745 {
5746 if (!CPUMIsHyperDebugStateActive(pVCpu))
5747 {
5748 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5749 uint32_t u32Val;
5750 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5751 pMixedCtx->dr[7] = u32Val;
5752 }
5753
5754 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
5755 }
5756 return VINF_SUCCESS;
5757}
5758
5759
5760/**
5761 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
5762 *
5763 * @returns VBox status code.
5764 * @param pVCpu Pointer to the VMCPU.
5765 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5766 * out-of-sync. Make sure to update the required fields
5767 * before using them.
5768 *
5769 * @remarks No-long-jump zone!!!
5770 */
5771static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5772{
5773 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5774 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
5775 return VINF_SUCCESS;
5776}
5777
5778
5779/**
5780 * Saves the entire guest state from the currently active VMCS into the
5781 * guest-CPU context. This essentially VMREADs all guest-data.
5782 *
5783 * @returns VBox status code.
5784 * @param pVCpu Pointer to the VMCPU.
5785 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5786 * out-of-sync. Make sure to update the required fields
5787 * before using them.
5788 */
5789static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5790{
5791 Assert(pVCpu);
5792 Assert(pMixedCtx);
5793
5794 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5795 return VINF_SUCCESS;
5796
5797 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5798 again on the ring-3 callback path, there is no real need to. */
5799 if (VMMRZCallRing3IsEnabled(pVCpu))
5800 VMMR0LogFlushDisable(pVCpu);
5801 else
5802 Assert(VMMR0IsLogFlushDisabled(pVCpu));
5803 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5804
5805 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5806 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5807
5808 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5809 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5810
5811 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5812 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5813
5814 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5815 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5816
5817 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5818 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5819
5820 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5821 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5822
5823 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5824 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5825
5826 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5827 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5828
5829 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5830 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5831
5832 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5833 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5834
5835 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5836 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5837
5838 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5839 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5840
5841 if (VMMRZCallRing3IsEnabled(pVCpu))
5842 VMMR0LogFlushEnable(pVCpu);
5843
5844 return rc;
5845}
5846
5847
5848/**
5849 * Check per-VM and per-VCPU force flag actions that require us to go back to
5850 * ring-3 for one reason or another.
5851 *
5852 * @returns VBox status code (information status code included).
5853 * @retval VINF_SUCCESS if we don't have any actions that require going back to
5854 * ring-3.
5855 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5856 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5857 * interrupts)
5858 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5859 * all EMTs to be in ring-3.
5860 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5861 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5862 * to the EM loop.
5863 *
5864 * @param pVM Pointer to the VM.
5865 * @param pVCpu Pointer to the VMCPU.
5866 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5867 * out-of-sync. Make sure to update the required fields
5868 * before using them.
5869 */
5870static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5871{
5872 Assert(VMMRZCallRing3IsEnabled(pVCpu));
5873
5874 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5875 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5876 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5877 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5878 {
5879 /* We need the control registers now, make sure the guest-CPU context is updated. */
5880 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5881 AssertRCReturn(rc3, rc3);
5882
5883 /* Pending HM CR3 sync. */
5884 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5885 {
5886 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5887 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
5888 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5889 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5890 }
5891
5892 /* Pending HM PAE PDPEs. */
5893 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5894 {
5895 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5896 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5897 }
5898
5899 /* Pending PGM C3 sync. */
5900 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5901 {
5902 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
5903 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5904 if (rc2 != VINF_SUCCESS)
5905 {
5906 AssertRC(rc2);
5907 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5908 return rc2;
5909 }
5910 }
5911
5912 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5913 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5914 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5915 {
5916 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5917 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5918 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5919 return rc2;
5920 }
5921
5922 /* Pending VM request packets, such as hardware interrupts. */
5923 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5924 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5925 {
5926 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5927 return VINF_EM_PENDING_REQUEST;
5928 }
5929
5930 /* Pending PGM pool flushes. */
5931 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5932 {
5933 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5934 return VINF_PGM_POOL_FLUSH_PENDING;
5935 }
5936
5937 /* Pending DMA requests. */
5938 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5939 {
5940 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5941 return VINF_EM_RAW_TO_R3;
5942 }
5943 }
5944
5945 return VINF_SUCCESS;
5946}
5947
5948
5949/**
5950 * Converts any TRPM trap into a pending HM event. This is typically used when
5951 * entering from ring-3 (not longjmp returns).
5952 *
5953 * @param pVCpu Pointer to the VMCPU.
5954 */
5955static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5956{
5957 Assert(TRPMHasTrap(pVCpu));
5958 Assert(!pVCpu->hm.s.Event.fPending);
5959
5960 uint8_t uVector;
5961 TRPMEVENT enmTrpmEvent;
5962 RTGCUINT uErrCode;
5963 RTGCUINTPTR GCPtrFaultAddress;
5964 uint8_t cbInstr;
5965
5966 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5967 AssertRC(rc);
5968
5969 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5970 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
5971 if (enmTrpmEvent == TRPM_TRAP)
5972 {
5973 switch (uVector)
5974 {
5975 case X86_XCPT_BP:
5976 case X86_XCPT_OF:
5977 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5978 break;
5979
5980 case X86_XCPT_PF:
5981 case X86_XCPT_DF:
5982 case X86_XCPT_TS:
5983 case X86_XCPT_NP:
5984 case X86_XCPT_SS:
5985 case X86_XCPT_GP:
5986 case X86_XCPT_AC:
5987 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5988 /* no break! */
5989 default:
5990 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5991 break;
5992 }
5993 }
5994 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5995 {
5996 if (uVector == X86_XCPT_NMI)
5997 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5998 else
5999 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6000 }
6001 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6002 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6003 else
6004 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6005
6006 rc = TRPMResetTrap(pVCpu);
6007 AssertRC(rc);
6008 Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6009 u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6010
6011 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6012 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6013}
6014
6015
6016/**
6017 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6018 * VT-x to execute any instruction.
6019 *
6020 * @param pvCpu Pointer to the VMCPU.
6021 */
6022static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6023{
6024 Assert(pVCpu->hm.s.Event.fPending);
6025
6026 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6027 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
6028 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
6029 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6030
6031 /* If a trap was already pending, we did something wrong! */
6032 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6033
6034 TRPMEVENT enmTrapType;
6035 switch (uVectorType)
6036 {
6037 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6038 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6039 enmTrapType = TRPM_HARDWARE_INT;
6040 break;
6041
6042 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6043 enmTrapType = TRPM_SOFTWARE_INT;
6044 break;
6045
6046 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6047 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6048 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6049 enmTrapType = TRPM_TRAP;
6050 break;
6051
6052 default:
6053 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6054 enmTrapType = TRPM_32BIT_HACK;
6055 break;
6056 }
6057
6058 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6059
6060 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6061 AssertRC(rc);
6062
6063 if (fErrorCodeValid)
6064 TRPMSetErrorCode(pVCpu, uErrorCode);
6065
6066 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6067 && uVector == X86_XCPT_PF)
6068 {
6069 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6070 }
6071 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6072 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6073 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6074 {
6075 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6076 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6077 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6078 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6079 }
6080 pVCpu->hm.s.Event.fPending = false;
6081}
6082
6083
6084/**
6085 * Does the necessary state syncing before returning to ring-3 for any reason
6086 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6087 *
6088 * @returns VBox status code.
6089 * @param pVM Pointer to the VM.
6090 * @param pVCpu Pointer to the VMCPU.
6091 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6092 * be out-of-sync. Make sure to update the required
6093 * fields before using them.
6094 * @param fSaveGuestState Whether to save the guest state or not.
6095 *
6096 * @remarks No-long-jmp zone!!!
6097 */
6098static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6099{
6100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6101 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6102
6103 RTCPUID idCpu = RTMpCpuId();
6104 Log4Func(("HostCpuId=%u\n", idCpu));
6105
6106 /* Save the guest state if necessary. */
6107 if ( fSaveGuestState
6108 && pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6109 {
6110 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6111 AssertRCReturn(rc, rc);
6112 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6113 }
6114
6115 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6116 if (CPUMIsGuestFPUStateActive(pVCpu))
6117 {
6118 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6119 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6120 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
6121 }
6122
6123 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6124#ifdef VBOX_STRICT
6125 if (CPUMIsHyperDebugStateActive(pVCpu))
6126 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6127#endif
6128 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6129 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
6130 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6131 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6132
6133#if HC_ARCH_BITS == 64
6134 /* Restore host-state bits that VT-x only restores partially. */
6135 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6136 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6137 {
6138 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6139 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6140 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6141 }
6142#endif
6143
6144 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6145 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6146 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6147 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6148 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6149 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6150 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6151 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6152
6153 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6154
6155 /** @todo This kinda defeats the purpose of having preemption hooks.
6156 * The problem is, deregistering the hooks should be moved to a place that
6157 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6158 * context.
6159 */
6160 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6161 {
6162 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6163 AssertRCReturn(rc, rc);
6164
6165 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6166 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6167 }
6168 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6169 NOREF(idCpu);
6170
6171 return VINF_SUCCESS;
6172}
6173
6174
6175/**
6176 * Leaves the VT-x session.
6177 *
6178 * @returns VBox status code.
6179 * @param pVM Pointer to the VM.
6180 * @param pVCpu Pointer to the VMCPU.
6181 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6182 * out-of-sync. Make sure to update the required fields
6183 * before using them.
6184 *
6185 * @remarks No-long-jmp zone!!!
6186 */
6187DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6188{
6189 HM_DISABLE_PREEMPT_IF_NEEDED();
6190 HMVMX_ASSERT_CPU_SAFE();
6191 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6192 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6193
6194 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6195 and done this from the VMXR0ThreadCtxCallback(). */
6196 if (!pVCpu->hm.s.fLeaveDone)
6197 {
6198 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6199 AssertRCReturn(rc2, rc2);
6200 pVCpu->hm.s.fLeaveDone = true;
6201 }
6202
6203 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6204 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6205 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6206 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6207 VMMR0ThreadCtxHooksDeregister(pVCpu);
6208
6209 /* Leave HM context. This takes care of local init (term). */
6210 int rc = HMR0LeaveCpu(pVCpu);
6211
6212 HM_RESTORE_PREEMPT_IF_NEEDED();
6213
6214 return rc;
6215}
6216
6217
6218/**
6219 * Does the necessary state syncing before doing a longjmp to ring-3.
6220 *
6221 * @returns VBox status code.
6222 * @param pVM Pointer to the VM.
6223 * @param pVCpu Pointer to the VMCPU.
6224 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6225 * out-of-sync. Make sure to update the required fields
6226 * before using them.
6227 *
6228 * @remarks No-long-jmp zone!!!
6229 */
6230DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6231{
6232 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6233}
6234
6235
6236/**
6237 * Take necessary actions before going back to ring-3.
6238 *
6239 * An action requires us to go back to ring-3. This function does the necessary
6240 * steps before we can safely return to ring-3. This is not the same as longjmps
6241 * to ring-3, this is voluntary and prepares the guest so it may continue
6242 * executing outside HM (recompiler/IEM).
6243 *
6244 * @returns VBox status code.
6245 * @param pVM Pointer to the VM.
6246 * @param pVCpu Pointer to the VMCPU.
6247 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6248 * out-of-sync. Make sure to update the required fields
6249 * before using them.
6250 * @param rcExit The reason for exiting to ring-3. Can be
6251 * VINF_VMM_UNKNOWN_RING3_CALL.
6252 */
6253static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6254{
6255 Assert(pVM);
6256 Assert(pVCpu);
6257 Assert(pMixedCtx);
6258 HMVMX_ASSERT_PREEMPT_SAFE();
6259
6260 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6261 {
6262 /* We've done what is required in hmR0VmxExitErrInvalidGuestState(). We're not going to continue guest execution... */
6263 return VINF_SUCCESS;
6264 }
6265 else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6266 {
6267 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6268 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6269 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6270 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6271 return VINF_SUCCESS;
6272 }
6273
6274 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6275 VMMRZCallRing3Disable(pVCpu);
6276 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6277
6278 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6279 if (pVCpu->hm.s.Event.fPending)
6280 {
6281 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6282 Assert(!pVCpu->hm.s.Event.fPending);
6283 }
6284
6285 /* Save guest state and restore host state bits. */
6286 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6287 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6288
6289 /* Sync recompiler state. */
6290 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6291 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6292 | CPUM_CHANGED_LDTR
6293 | CPUM_CHANGED_GDTR
6294 | CPUM_CHANGED_IDTR
6295 | CPUM_CHANGED_TR
6296 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6297 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6298 if ( pVM->hm.s.fNestedPaging
6299 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6300 {
6301 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6302 }
6303
6304 /*
6305 * Clear the X86_EFL_TF if necessary.
6306 */
6307 if (pVCpu->hm.s.fClearTrapFlag)
6308 {
6309 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6310 pMixedCtx->eflags.Bits.u1TF = 0;
6311 pVCpu->hm.s.fClearTrapFlag = false;
6312 }
6313 /** @todo there seems to be issues with the resume flag when the monitor trap
6314 * flag is pending without being used. Seen early in bios init when
6315 * accessing APIC page in prot mode. */
6316
6317 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6318 if (rcExit != VINF_EM_RAW_INTERRUPT)
6319 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
6320
6321 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6322
6323 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6324 VMMRZCallRing3RemoveNotification(pVCpu);
6325 VMMRZCallRing3Enable(pVCpu);
6326
6327 return rc;
6328}
6329
6330
6331/**
6332 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6333 * longjump to ring-3 and possibly get preempted.
6334 *
6335 * @returns VBox status code.
6336 * @param pVCpu Pointer to the VMCPU.
6337 * @param enmOperation The operation causing the ring-3 longjump.
6338 * @param pvUser Opaque pointer to the guest-CPU context. The data
6339 * may be out-of-sync. Make sure to update the required
6340 * fields before using them.
6341 *
6342 * @remarks Must never be called with @a enmOperation ==
6343 * VMMCALLRING3_VM_R0_ASSERTION. We can't assert it here because if it
6344 * it -does- get called with VMMCALLRING3_VM_R0_ASSERTION, we'll end up
6345 * with an infinite recursion.
6346 */
6347DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6348{
6349 /* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6350 Assert(pVCpu);
6351 Assert(pvUser);
6352 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6353 HMVMX_ASSERT_PREEMPT_SAFE();
6354
6355 VMMRZCallRing3Disable(pVCpu);
6356 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6357
6358 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6359 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6360 AssertRCReturn(rc, rc);
6361
6362 VMMRZCallRing3Enable(pVCpu);
6363 return VINF_SUCCESS;
6364}
6365
6366
6367/**
6368 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6369 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6370 *
6371 * @param pVCpu Pointer to the VMCPU.
6372 */
6373DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6374{
6375 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6376 {
6377 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6378 {
6379 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6380 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6381 AssertRC(rc);
6382 }
6383 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6384}
6385
6386
6387/**
6388 * Evaluates the event to be delivered to the guest and sets it as the pending
6389 * event.
6390 *
6391 * @param pVCpu Pointer to the VMCPU.
6392 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6393 * out-of-sync. Make sure to update the required fields
6394 * before using them.
6395 */
6396static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6397{
6398 Assert(!pVCpu->hm.s.Event.fPending);
6399
6400 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6401 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6402 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6403 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6404
6405 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6406 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6407 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6408 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6409 Assert(!TRPMHasTrap(pVCpu));
6410
6411 /** @todo SMI. SMIs take priority over NMIs. */
6412 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6413 {
6414 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6415 if ( !fBlockMovSS
6416 && !fBlockSti)
6417 {
6418 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6419 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6420 uint32_t u32IntrInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6421 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6422
6423 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6424 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6425 }
6426 else
6427 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6428 }
6429 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6430 && !pVCpu->hm.s.fSingleInstruction)
6431 {
6432 /*
6433 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6434 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6435 * evaluated here and not set as pending, solely based on the force-flags.
6436 */
6437 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6438 AssertRC(rc);
6439 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6440 if ( !fBlockInt
6441 && !fBlockSti
6442 && !fBlockMovSS)
6443 {
6444 uint8_t u8Interrupt;
6445 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6446 if (RT_SUCCESS(rc))
6447 {
6448 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6449 uint32_t u32IntrInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6450 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6451
6452 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6453 }
6454 else
6455 {
6456 /** @todo Does this actually happen? If not turn it into an assertion. */
6457 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6458 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6459 }
6460 }
6461 else
6462 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6463 }
6464}
6465
6466
6467/**
6468 * Injects any pending events into the guest if the guest is in a state to
6469 * receive them.
6470 *
6471 * @returns VBox status code (informational status codes included).
6472 * @param pVCpu Pointer to the VMCPU.
6473 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6474 * out-of-sync. Make sure to update the required fields
6475 * before using them.
6476 */
6477static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6478{
6479 HMVMX_ASSERT_PREEMPT_SAFE();
6480 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6481
6482 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6483 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6484 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6485 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6486
6487 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6488 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6489 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6490 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6491 Assert(!TRPMHasTrap(pVCpu));
6492
6493 int rc = VINF_SUCCESS;
6494 if (pVCpu->hm.s.Event.fPending)
6495 {
6496#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6497 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6498 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6499 {
6500 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6501 AssertRCReturn(rc, rc);
6502 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6503 Assert(!fBlockInt);
6504 Assert(!fBlockSti);
6505 Assert(!fBlockMovSS);
6506 }
6507 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6508 {
6509 Assert(!fBlockSti);
6510 Assert(!fBlockMovSS);
6511 }
6512#endif
6513 Log4(("Injecting pending event vcpu[%RU32] u64IntrInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo));
6514 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6515 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6516 AssertRCReturn(rc, rc);
6517
6518 /* Update the interruptibility-state as it could have been changed by
6519 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6520 fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6521 fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6522
6523#ifdef VBOX_WITH_STATISTICS
6524 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6525 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6526 else
6527 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6528#endif
6529 }
6530
6531 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6532 int rc2 = VINF_SUCCESS;
6533 if ( fBlockSti
6534 || fBlockMovSS)
6535 {
6536 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6537 {
6538 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6539 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6540 {
6541 /*
6542 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6543 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6544 * See Intel spec. 27.3.4 "Saving Non-Register State".
6545 */
6546 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6547 AssertRCReturn(rc, rc);
6548 }
6549 }
6550 else
6551 {
6552 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6553 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6554 uIntrState = 0;
6555 }
6556 }
6557
6558 /*
6559 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6560 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6561 */
6562 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6563 AssertRC(rc2);
6564
6565 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6566 return rc;
6567}
6568
6569
6570/**
6571 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6572 *
6573 * @param pVCpu Pointer to the VMCPU.
6574 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6575 * out-of-sync. Make sure to update the required fields
6576 * before using them.
6577 */
6578DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6579{
6580 uint32_t u32IntrInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6581 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6582}
6583
6584
6585/**
6586 * Injects a double-fault (#DF) exception into the VM.
6587 *
6588 * @returns VBox status code (informational status code included).
6589 * @param pVCpu Pointer to the VMCPU.
6590 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6591 * out-of-sync. Make sure to update the required fields
6592 * before using them.
6593 */
6594DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6595{
6596 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6597 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6598 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6599 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6600 puIntrState);
6601}
6602
6603
6604/**
6605 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6606 *
6607 * @param pVCpu Pointer to the VMCPU.
6608 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6609 * out-of-sync. Make sure to update the required fields
6610 * before using them.
6611 */
6612DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6613{
6614 uint32_t u32IntrInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6615 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6616 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6617}
6618
6619
6620/**
6621 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6622 *
6623 * @param pVCpu Pointer to the VMCPU.
6624 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6625 * out-of-sync. Make sure to update the required fields
6626 * before using them.
6627 * @param cbInstr The value of RIP that is to be pushed on the guest
6628 * stack.
6629 */
6630DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6631{
6632 uint32_t u32IntrInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6633 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6634 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6635}
6636
6637
6638/**
6639 * Injects a general-protection (#GP) fault into the VM.
6640 *
6641 * @returns VBox status code (informational status code included).
6642 * @param pVCpu Pointer to the VMCPU.
6643 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6644 * out-of-sync. Make sure to update the required fields
6645 * before using them.
6646 * @param u32ErrorCode The error code associated with the #GP.
6647 */
6648DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6649 uint32_t *puIntrState)
6650{
6651 uint32_t u32IntrInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6652 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6653 if (fErrorCodeValid)
6654 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6655 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6656 puIntrState);
6657}
6658
6659
6660/**
6661 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6662 *
6663 * @param pVCpu Pointer to the VMCPU.
6664 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6665 * out-of-sync. Make sure to update the required fields
6666 * before using them.
6667 * @param uVector The software interrupt vector number.
6668 * @param cbInstr The value of RIP that is to be pushed on the guest
6669 * stack.
6670 */
6671DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6672{
6673 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6674 if ( uVector == X86_XCPT_BP
6675 || uVector == X86_XCPT_OF)
6676 {
6677 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6678 }
6679 else
6680 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6681 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6682}
6683
6684
6685/**
6686 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6687 * stack.
6688 *
6689 * @returns VBox status code (information status code included).
6690 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6691 * @param pVM Pointer to the VM.
6692 * @param pMixedCtx Pointer to the guest-CPU context.
6693 * @param uValue The value to push to the guest stack.
6694 */
6695DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6696{
6697 /*
6698 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6699 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6700 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6701 */
6702 if (pMixedCtx->sp == 1)
6703 return VINF_EM_RESET;
6704 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6705 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6706 AssertRCReturn(rc, rc);
6707 return rc;
6708}
6709
6710
6711/**
6712 * Injects an event into the guest upon VM-entry by updating the relevant fields
6713 * in the VM-entry area in the VMCS.
6714 *
6715 * @returns VBox status code (informational error codes included).
6716 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6717 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6718 *
6719 * @param pVCpu Pointer to the VMCPU.
6720 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6721 * be out-of-sync. Make sure to update the required
6722 * fields before using them.
6723 * @param u64IntrInfo The VM-entry interruption-information field.
6724 * @param cbInstr The VM-entry instruction length in bytes (for
6725 * software interrupts, exceptions and privileged
6726 * software exceptions).
6727 * @param u32ErrCode The VM-entry exception error code.
6728 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6729 * @param puIntrState Pointer to the current guest interruptibility-state.
6730 * This interruptibility-state will be updated if
6731 * necessary. This cannot not be NULL.
6732 *
6733 * @remarks Requires CR0!
6734 * @remarks No-long-jump zone!!!
6735 */
6736static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6737 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6738{
6739 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6740 AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6741 Assert(puIntrState);
6742 uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6743
6744 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6745 const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6746
6747#ifdef VBOX_STRICT
6748 /* Validate the error-code-valid bit for hardware exceptions. */
6749 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6750 {
6751 switch (uVector)
6752 {
6753 case X86_XCPT_PF:
6754 case X86_XCPT_DF:
6755 case X86_XCPT_TS:
6756 case X86_XCPT_NP:
6757 case X86_XCPT_SS:
6758 case X86_XCPT_GP:
6759 case X86_XCPT_AC:
6760 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6761 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6762 /* fallthru */
6763 default:
6764 break;
6765 }
6766 }
6767#endif
6768
6769 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
6770 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6771 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6772
6773 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6774
6775 /* We require CR0 to check if the guest is in real-mode. */
6776 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6777 AssertRCReturn(rc, rc);
6778
6779 /*
6780 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6781 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6782 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6783 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6784 */
6785 if (CPUMIsGuestInRealModeEx(pMixedCtx))
6786 {
6787 PVM pVM = pVCpu->CTX_SUFF(pVM);
6788 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6789 {
6790 Assert(PDMVmmDevHeapIsEnabled(pVM));
6791 Assert(pVM->hm.s.vmx.pRealModeTSS);
6792
6793 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6794 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6795 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6796 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6797 AssertRCReturn(rc, rc);
6798 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6799
6800 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6801 const size_t cbIdtEntry = sizeof(X86IDTR16);
6802 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6803 {
6804 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6805 if (uVector == X86_XCPT_DF)
6806 return VINF_EM_RESET;
6807 else if (uVector == X86_XCPT_GP)
6808 {
6809 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6810 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6811 }
6812
6813 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6814 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6815 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
6816 }
6817
6818 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6819 uint16_t uGuestIp = pMixedCtx->ip;
6820 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6821 {
6822 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
6823 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
6824 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6825 }
6826 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6827 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6828
6829 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6830 X86IDTR16 IdtEntry;
6831 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6832 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6833 AssertRCReturn(rc, rc);
6834
6835 /* Construct the stack frame for the interrupt/exception handler. */
6836 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6837 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6838 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6839 AssertRCReturn(rc, rc);
6840
6841 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
6842 if (rc == VINF_SUCCESS)
6843 {
6844 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
6845 pMixedCtx->rip = IdtEntry.offSel;
6846 pMixedCtx->cs.Sel = IdtEntry.uSel;
6847 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6848 if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6849 && uVector == X86_XCPT_PF)
6850 {
6851 pMixedCtx->cr2 = GCPtrFaultAddress;
6852 }
6853
6854 /* If any other guest-state bits are changed here, make sure to update
6855 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6856 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS
6857 | HM_CHANGED_GUEST_RIP
6858 | HM_CHANGED_GUEST_RFLAGS
6859 | HM_CHANGED_GUEST_RSP;
6860
6861 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6862 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6863 {
6864 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6865 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6866 Log4(("Clearing inhibition due to STI.\n"));
6867 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6868 }
6869 Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6870
6871 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
6872 it, if we are returning to ring-3 before executing guest code. */
6873 pVCpu->hm.s.Event.fPending = false;
6874 }
6875 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6876 return rc;
6877 }
6878 else
6879 {
6880 /*
6881 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6882 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6883 */
6884 u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6885 }
6886 }
6887
6888 /* Validate. */
6889 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6890 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6891 Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6892
6893 /* Inject. */
6894 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6895 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6896 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6897 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6898
6899 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6900 && uVector == X86_XCPT_PF)
6901 {
6902 pMixedCtx->cr2 = GCPtrFaultAddress;
6903 }
6904
6905 Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6906 u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6907
6908 AssertRCReturn(rc, rc);
6909 return rc;
6910}
6911
6912
6913/**
6914 * Clears the current event in the VMCS.
6915 *
6916 * @returns VBox status code.
6917 * @param pVCpu Pointer to the VMCPU.
6918 *
6919 * @remarks Use this function only to clear events that have not yet been
6920 * delivered to the guest but are injected in the VMCS!
6921 * @remarks No-long-jump zone!!!
6922 */
6923static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
6924{
6925 if (!pVCpu->hm.s.Event.fPending)
6926 return;
6927
6928#ifdef VBOX_STRICT
6929 uint32_t u32EntryInfo;
6930 int rc2 = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
6931 AssertRC(rc2);
6932 Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
6933#endif
6934
6935 /* Clear the entry-interruption field (including the valid bit). */
6936 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
6937 AssertRC(rc);
6938
6939 /* Clear the pending debug exception field. */
6940 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
6941 AssertRC(rc);
6942}
6943
6944
6945/**
6946 * Enters the VT-x session.
6947 *
6948 * @returns VBox status code.
6949 * @param pVM Pointer to the VM.
6950 * @param pVCpu Pointer to the VMCPU.
6951 * @param pCpu Pointer to the CPU info struct.
6952 */
6953VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
6954{
6955 AssertPtr(pVM);
6956 AssertPtr(pVCpu);
6957 Assert(pVM->hm.s.vmx.fSupported);
6958 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6959 NOREF(pCpu);
6960
6961 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6962 Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
6963
6964#ifdef VBOX_STRICT
6965 /* Make sure we're in VMX root mode. */
6966 RTCCUINTREG u32HostCR4 = ASMGetCR4();
6967 if (!(u32HostCR4 & X86_CR4_VMXE))
6968 {
6969 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6970 return VERR_VMX_X86_CR4_VMXE_CLEARED;
6971 }
6972#endif
6973
6974 /*
6975 * Load the VCPU's VMCS as the current (and active) one.
6976 */
6977 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
6978 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6979 if (RT_FAILURE(rc))
6980 return rc;
6981
6982 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
6983 pVCpu->hm.s.fLeaveDone = false;
6984 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
6985
6986 return VINF_SUCCESS;
6987}
6988
6989
6990/**
6991 * The thread-context callback (only on platforms which support it).
6992 *
6993 * @param enmEvent The thread-context event.
6994 * @param pVCpu Pointer to the VMCPU.
6995 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
6996 * @thread EMT.
6997 */
6998VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
6999{
7000 switch (enmEvent)
7001 {
7002 case RTTHREADCTXEVENT_PREEMPTING:
7003 {
7004 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7005 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7006 VMCPU_ASSERT_EMT(pVCpu);
7007
7008 PVM pVM = pVCpu->CTX_SUFF(pVM);
7009 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7010
7011 /* No longjmps (logger flushes, locks) in this fragile context. */
7012 VMMRZCallRing3Disable(pVCpu);
7013 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7014
7015 /*
7016 * Restore host-state (FPU, debug etc.)
7017 */
7018 if (!pVCpu->hm.s.fLeaveDone)
7019 {
7020 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7021 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7022 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7023 pVCpu->hm.s.fLeaveDone = true;
7024 }
7025
7026 /* Leave HM context, takes care of local init (term). */
7027 int rc = HMR0LeaveCpu(pVCpu);
7028 AssertRC(rc); NOREF(rc);
7029
7030 /* Restore longjmp state. */
7031 VMMRZCallRing3Enable(pVCpu);
7032 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7033 break;
7034 }
7035
7036 case RTTHREADCTXEVENT_RESUMED:
7037 {
7038 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7039 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7040 VMCPU_ASSERT_EMT(pVCpu);
7041
7042 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7043 VMMRZCallRing3Disable(pVCpu);
7044 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7045
7046 /* Initialize the bare minimum state required for HM. This takes care of
7047 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7048 int rc = HMR0EnterCpu(pVCpu);
7049 AssertRC(rc);
7050 Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7051
7052 /* Load the active VMCS as the current one. */
7053 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7054 {
7055 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7056 AssertRC(rc); NOREF(rc);
7057 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7058 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7059 }
7060 pVCpu->hm.s.fLeaveDone = false;
7061
7062 /* Restore longjmp state. */
7063 VMMRZCallRing3Enable(pVCpu);
7064 break;
7065 }
7066
7067 default:
7068 break;
7069 }
7070}
7071
7072
7073/**
7074 * Saves the host state in the VMCS host-state.
7075 * Sets up the VM-exit MSR-load area.
7076 *
7077 * The CPU state will be loaded from these fields on every successful VM-exit.
7078 *
7079 * @returns VBox status code.
7080 * @param pVM Pointer to the VM.
7081 * @param pVCpu Pointer to the VMCPU.
7082 *
7083 * @remarks No-long-jump zone!!!
7084 */
7085static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7086{
7087 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7088
7089 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
7090 return VINF_SUCCESS;
7091
7092 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7093 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7094
7095 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7096 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7097
7098 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7099 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7100
7101 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
7102 return rc;
7103}
7104
7105
7106/**
7107 * Saves the host state in the VMCS host-state.
7108 *
7109 * @returns VBox status code.
7110 * @param pVM Pointer to the VM.
7111 * @param pVCpu Pointer to the VMCPU.
7112 *
7113 * @remarks No-long-jump zone!!!
7114 */
7115VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7116{
7117 AssertPtr(pVM);
7118 AssertPtr(pVCpu);
7119
7120 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7121
7122 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7123 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7124 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7125 return hmR0VmxSaveHostState(pVM, pVCpu);
7126}
7127
7128
7129/**
7130 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7131 * loaded from these fields on every successful VM-entry.
7132 *
7133 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7134 * Sets up the VM-entry controls.
7135 * Sets up the appropriate VMX non-root function to execute guest code based on
7136 * the guest CPU mode.
7137 *
7138 * @returns VBox status code.
7139 * @param pVM Pointer to the VM.
7140 * @param pVCpu Pointer to the VMCPU.
7141 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7142 * out-of-sync. Make sure to update the required fields
7143 * before using them.
7144 *
7145 * @remarks No-long-jump zone!!!
7146 */
7147static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7148{
7149 AssertPtr(pVM);
7150 AssertPtr(pVCpu);
7151 AssertPtr(pMixedCtx);
7152 HMVMX_ASSERT_PREEMPT_SAFE();
7153
7154#ifdef LOG_ENABLED
7155 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7156 * probably not initialized yet? Anyway this will do for now.
7157 *
7158 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7159 * interface and disable ring-3 calls when thread-context hooks are not
7160 * available. */
7161 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7162 VMMR0LogFlushDisable(pVCpu);
7163#endif
7164
7165 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7166
7167 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7168
7169 /* Determine real-on-v86 mode. */
7170 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7171 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7172 && CPUMIsGuestInRealModeEx(pMixedCtx))
7173 {
7174 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7175 }
7176
7177 /*
7178 * Load the guest-state into the VMCS.
7179 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7180 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7181 */
7182 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7183 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7184
7185 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7186 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7187 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7188
7189 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7190 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7191 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7192
7193 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7194 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7195
7196 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7197 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7198
7199 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7200 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7201 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7202
7203 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7204 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7205
7206 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7207 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7208
7209 /*
7210 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7211 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7212 */
7213 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7214 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7215
7216 /* Clear any unused and reserved bits. */
7217 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
7218
7219#ifdef LOG_ENABLED
7220 /* Only reenable log-flushing if the caller has it enabled. */
7221 if (!fCallerDisabledLogFlush)
7222 VMMR0LogFlushEnable(pVCpu);
7223#endif
7224
7225 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7226 return rc;
7227}
7228
7229
7230/**
7231 * Loads the state shared between the host and guest into the VMCS.
7232 *
7233 * @param pVM Pointer to the VM.
7234 * @param pVCpu Pointer to the VMCPU.
7235 * @param pCtx Pointer to the guest-CPU context.
7236 *
7237 * @remarks No-long-jump zone!!!
7238 */
7239static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7240{
7241 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7242 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7243
7244 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
7245 {
7246 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7247 AssertRC(rc);
7248 }
7249
7250 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
7251 {
7252 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7253 AssertRC(rc);
7254
7255 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7256 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
7257 {
7258 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7259 AssertRC(rc);
7260 }
7261 }
7262
7263 AssertMsg(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE), ("fContextUseFlags=%#x\n",
7264 pVCpu->hm.s.fContextUseFlags));
7265}
7266
7267
7268/**
7269 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7270 *
7271 * @param pVM Pointer to the VM.
7272 * @param pVCpu Pointer to the VMCPU.
7273 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7274 * out-of-sync. Make sure to update the required fields
7275 * before using them.
7276 */
7277DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7278{
7279 HMVMX_ASSERT_PREEMPT_SAFE();
7280
7281 Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7282#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7283 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
7284#endif
7285
7286 if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7287 {
7288 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7289 AssertRC(rc);
7290 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7291 }
7292 else if (pVCpu->hm.s.fContextUseFlags)
7293 {
7294 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7295 AssertRC(rc);
7296 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7297 }
7298
7299 /* All the guest state bits should be loaded except maybe the host context and shared host/guest bits. */
7300 AssertMsg( !(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST)
7301 || !(pVCpu->hm.s.fContextUseFlags & ~(HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE)),
7302 ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7303
7304#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7305 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7306 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7307 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7308#endif
7309}
7310
7311
7312/**
7313 * Does the preparations before executing guest code in VT-x.
7314 *
7315 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7316 * recompiler. We must be cautious what we do here regarding committing
7317 * guest-state information into the VMCS assuming we assuredly execute the
7318 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7319 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7320 * so that the recompiler can (and should) use them when it resumes guest
7321 * execution. Otherwise such operations must be done when we can no longer
7322 * exit to ring-3.
7323 *
7324 * @returns Strict VBox status code.
7325 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7326 * have been disabled.
7327 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7328 * double-fault into the guest.
7329 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7330 *
7331 * @param pVM Pointer to the VM.
7332 * @param pVCpu Pointer to the VMCPU.
7333 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7334 * out-of-sync. Make sure to update the required fields
7335 * before using them.
7336 * @param pVmxTransient Pointer to the VMX transient structure.
7337 *
7338 * @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7339 * interrupts will be disabled.
7340 */
7341static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7342{
7343 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7344
7345#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7346 PGMRZDynMapFlushAutoSet(pVCpu);
7347#endif
7348
7349 /* Check force flag actions that might require us to go back to ring-3. */
7350 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7351 if (rc != VINF_SUCCESS)
7352 return rc;
7353
7354#ifndef IEM_VERIFICATION_MODE_FULL
7355 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7356 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7357 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7358 {
7359 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7360 RTGCPHYS GCPhysApicBase;
7361 GCPhysApicBase = pMixedCtx->msrApicBase;
7362 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7363
7364 /* Unalias any existing mapping. */
7365 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7366 AssertRCReturn(rc, rc);
7367
7368 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7369 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7370 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7371 AssertRCReturn(rc, rc);
7372
7373 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7374 }
7375#endif /* !IEM_VERIFICATION_MODE_FULL */
7376
7377 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7378 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7379
7380 /*
7381 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7382 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7383 */
7384 if (TRPMHasTrap(pVCpu))
7385 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7386 else if (!pVCpu->hm.s.Event.fPending)
7387 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7388
7389 /*
7390 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7391 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7392 */
7393 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7394 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7395 {
7396 Assert(rc == VINF_EM_RESET);
7397 return rc;
7398 }
7399
7400 /*
7401 * No longjmps to ring-3 from this point on!!!
7402 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7403 * This also disables flushing of the R0-logger instance (if any).
7404 */
7405 VMMRZCallRing3Disable(pVCpu);
7406
7407 /*
7408 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7409 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7410 *
7411 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7412 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7413 *
7414 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7415 * executing guest code.
7416 */
7417 pVmxTransient->uEflags = ASMIntDisableFlags();
7418 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7419 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7420 {
7421 hmR0VmxClearEventVmcs(pVCpu);
7422 ASMSetFlags(pVmxTransient->uEflags);
7423 VMMRZCallRing3Enable(pVCpu);
7424 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7425 return VINF_EM_RAW_TO_R3;
7426 }
7427 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7428 {
7429 hmR0VmxClearEventVmcs(pVCpu);
7430 ASMSetFlags(pVmxTransient->uEflags);
7431 VMMRZCallRing3Enable(pVCpu);
7432 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7433 return VINF_EM_RAW_INTERRUPT;
7434 }
7435
7436 /* We've injected any pending events. This is really the point of no return (to ring-3). */
7437 pVCpu->hm.s.Event.fPending = false;
7438
7439 return VINF_SUCCESS;
7440}
7441
7442
7443/**
7444 * Prepares to run guest code in VT-x and we've committed to doing so. This
7445 * means there is no backing out to ring-3 or anywhere else at this
7446 * point.
7447 *
7448 * @param pVM Pointer to the VM.
7449 * @param pVCpu Pointer to the VMCPU.
7450 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7451 * out-of-sync. Make sure to update the required fields
7452 * before using them.
7453 * @param pVmxTransient Pointer to the VMX transient structure.
7454 *
7455 * @remarks Called with preemption disabled.
7456 * @remarks No-long-jump zone!!!
7457 */
7458static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7459{
7460 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7461 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7462 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7463
7464 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7465 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7466
7467 /*
7468 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7469 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7470 * Reload only the necessary state, the assertion will catch if other parts of the code
7471 * change.
7472 */
7473 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7474 {
7475 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7476 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7477 }
7478
7479 /*
7480 * Load the host state bits as we may've been preempted (only happens when
7481 * thread-context hooks are used).
7482 */
7483 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
7484 {
7485 /* This ASSUMES that pfnStartVM has been set up already. */
7486 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7487 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7488 AssertRC(rc);
7489 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7490 }
7491 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7492
7493 /*
7494 * Load the state shared between host and guest (FPU, debug).
7495 */
7496 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE)
7497 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7498 AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7499
7500 /*
7501 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7502 */
7503 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7504 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7505
7506 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7507 RTCPUID idCurrentCpu = pCpu->idCpu;
7508 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7509 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
7510 {
7511 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7512 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7513 }
7514
7515 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7516 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7517 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7518 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7519
7520 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7521
7522 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7523 to start executing. */
7524
7525#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7526 /*
7527 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7528 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7529 */
7530 if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7531 && !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7532 {
7533 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7534 uint64_t u64HostTscAux = 0;
7535 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7536 AssertRC(rc2);
7537 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7538 }
7539#endif
7540}
7541
7542
7543/**
7544 * Performs some essential restoration of state after running guest code in
7545 * VT-x.
7546 *
7547 * @param pVM Pointer to the VM.
7548 * @param pVCpu Pointer to the VMCPU.
7549 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7550 * out-of-sync. Make sure to update the required fields
7551 * before using them.
7552 * @param pVmxTransient Pointer to the VMX transient structure.
7553 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7554 *
7555 * @remarks Called with interrupts disabled, and returns with interrups enabled!
7556 *
7557 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7558 * unconditionally when it is safe to do so.
7559 */
7560static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7561{
7562 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7563
7564 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7565 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7566 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7567 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7568 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7569
7570 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7571 {
7572#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7573 /* Restore host's TSC_AUX. */
7574 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7575 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7576#endif
7577 /** @todo Find a way to fix hardcoding a guestimate. */
7578 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7579 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7580 }
7581
7582 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7583 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7584 Assert(!(ASMGetFlags() & X86_EFL_IF));
7585 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7586
7587 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7588 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7589 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7590 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7591
7592 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7593 uint32_t uExitReason;
7594 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7595 rc |= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7596 AssertRC(rc);
7597 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7598 pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7599
7600 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7601 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7602 {
7603 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7604 pVmxTransient->fVMEntryFailed));
7605 return;
7606 }
7607
7608 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7609 {
7610 /* Update the guest interruptibility-state from the VMCS. */
7611 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7612#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7613 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7614 AssertRC(rc);
7615#endif
7616 /*
7617 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7618 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7619 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7620 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7621 */
7622 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7623 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7624 {
7625 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7626 AssertRC(rc);
7627 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
7628 }
7629 }
7630}
7631
7632
7633
7634/**
7635 * Runs the guest code using VT-x the normal way.
7636 *
7637 * @returns VBox status code.
7638 * @param pVM Pointer to the VM.
7639 * @param pVCpu Pointer to the VMCPU.
7640 * @param pCtx Pointer to the guest-CPU context.
7641 *
7642 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
7643 * @remarks Called with preemption disabled.
7644 */
7645static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7646{
7647 VMXTRANSIENT VmxTransient;
7648 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7649 int rc = VERR_INTERNAL_ERROR_5;
7650 uint32_t cLoops = 0;
7651
7652 for (;; cLoops++)
7653 {
7654 Assert(!HMR0SuspendPending());
7655 HMVMX_ASSERT_CPU_SAFE();
7656
7657 /* Preparatory work for running guest code, this may force us to return
7658 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7659 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7660 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7661 if (rc != VINF_SUCCESS)
7662 break;
7663
7664 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7665 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7666 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7667
7668 /* Restore any residual host-state and save any bits shared between host
7669 and guest into the guest-CPU state. Re-enables interrupts! */
7670 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7671
7672 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7673 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7674 {
7675 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7676 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7677 return rc;
7678 }
7679
7680 /* Handle the VM-exit. */
7681 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7682 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7683 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7684 HMVMX_START_EXIT_DISPATCH_PROF();
7685#ifdef HMVMX_USE_FUNCTION_TABLE
7686 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7687#else
7688 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7689#endif
7690 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7691 if (rc != VINF_SUCCESS)
7692 break;
7693 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7694 {
7695 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7696 rc = VINF_EM_RAW_INTERRUPT;
7697 break;
7698 }
7699 }
7700
7701 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7702 return rc;
7703}
7704
7705
7706/**
7707 * Single steps guest code using VT-x.
7708 *
7709 * @returns VBox status code.
7710 * @param pVM Pointer to the VM.
7711 * @param pVCpu Pointer to the VMCPU.
7712 * @param pCtx Pointer to the guest-CPU context.
7713 *
7714 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7715 * @remarks Called with preemption disabled.
7716 */
7717static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7718{
7719 VMXTRANSIENT VmxTransient;
7720 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7721 int rc = VERR_INTERNAL_ERROR_5;
7722 uint32_t cLoops = 0;
7723 uint16_t uCsStart = pCtx->cs.Sel;
7724 uint64_t uRipStart = pCtx->rip;
7725
7726 for (;; cLoops++)
7727 {
7728 Assert(!HMR0SuspendPending());
7729 HMVMX_ASSERT_CPU_SAFE();
7730
7731 /* Preparatory work for running guest code, this may force us to return
7732 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7733 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7734 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7735 if (rc != VINF_SUCCESS)
7736 break;
7737
7738 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7739 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7740 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7741
7742 /* Restore any residual host-state and save any bits shared between host
7743 and guest into the guest-CPU state. Re-enables interrupts! */
7744 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7745
7746 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7747 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7748 {
7749 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7750 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7751 return rc;
7752 }
7753
7754 /* Handle the VM-exit. */
7755 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7756 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7757 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7758 HMVMX_START_EXIT_DISPATCH_PROF();
7759#ifdef HMVMX_USE_FUNCTION_TABLE
7760 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7761#else
7762 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7763#endif
7764 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7765 if (rc != VINF_SUCCESS)
7766 break;
7767 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7768 {
7769 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7770 rc = VINF_EM_RAW_INTERRUPT;
7771 break;
7772 }
7773
7774 /*
7775 * Did the RIP change, if so, consider it a single step.
7776 * Otherwise, make sure one of the TFs gets set.
7777 */
7778 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7779 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7780 AssertRCReturn(rc2, rc2);
7781 if ( pCtx->rip != uRipStart
7782 || pCtx->cs.Sel != uCsStart)
7783 {
7784 rc = VINF_EM_DBG_STEPPED;
7785 break;
7786 }
7787 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
7788 }
7789
7790 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7791 return rc;
7792}
7793
7794
7795/**
7796 * Runs the guest code using VT-x.
7797 *
7798 * @returns VBox status code.
7799 * @param pVM Pointer to the VM.
7800 * @param pVCpu Pointer to the VMCPU.
7801 * @param pCtx Pointer to the guest-CPU context.
7802 *
7803 * @remarks Called with preemption disabled.
7804 */
7805VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7806{
7807 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7808 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
7809 HMVMX_ASSERT_PREEMPT_SAFE();
7810
7811 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
7812
7813 int rc;
7814 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7815 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7816 else
7817 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7818
7819 if (rc == VERR_EM_INTERPRETER)
7820 rc = VINF_EM_RAW_EMULATE_INSTR;
7821 else if (rc == VINF_EM_RESET)
7822 rc = VINF_EM_TRIPLE_FAULT;
7823
7824 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7825 if (RT_FAILURE(rc2))
7826 {
7827 pVCpu->hm.s.u32HMError = rc;
7828 rc = rc2;
7829 }
7830 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
7831 return rc;
7832}
7833
7834
7835#ifndef HMVMX_USE_FUNCTION_TABLE
7836DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7837{
7838 int rc;
7839 switch (rcReason)
7840 {
7841 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7842 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7843 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7844 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7845 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7846 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7847 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7848 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7849 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7850 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7851 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7852 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7853 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7854 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7855 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7856 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7857 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7858 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7859 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7860 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7861 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7862 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7863 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7864 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7865 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7866 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7867 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7868 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7869 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7870 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7871 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7872 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7873 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7874
7875 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7876 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7877 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7878 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7879 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7880 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7881 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7882 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7883 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7884
7885 case VMX_EXIT_VMCALL:
7886 case VMX_EXIT_VMCLEAR:
7887 case VMX_EXIT_VMLAUNCH:
7888 case VMX_EXIT_VMPTRLD:
7889 case VMX_EXIT_VMPTRST:
7890 case VMX_EXIT_VMREAD:
7891 case VMX_EXIT_VMRESUME:
7892 case VMX_EXIT_VMWRITE:
7893 case VMX_EXIT_VMXOFF:
7894 case VMX_EXIT_VMXON:
7895 case VMX_EXIT_INVEPT:
7896 case VMX_EXIT_INVVPID:
7897 case VMX_EXIT_VMFUNC:
7898 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7899 break;
7900 default:
7901 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7902 break;
7903 }
7904 return rc;
7905}
7906#endif
7907
7908#ifdef DEBUG
7909/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7910# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7911 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7912
7913# define HMVMX_ASSERT_PREEMPT_CPUID() \
7914 do \
7915 { \
7916 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7917 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7918 } while (0)
7919
7920# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7921 do { \
7922 AssertPtr(pVCpu); \
7923 AssertPtr(pMixedCtx); \
7924 AssertPtr(pVmxTransient); \
7925 Assert(pVmxTransient->fVMEntryFailed == false); \
7926 Assert(ASMIntAreEnabled()); \
7927 HMVMX_ASSERT_PREEMPT_SAFE(); \
7928 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7929 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7930 HMVMX_ASSERT_PREEMPT_SAFE(); \
7931 if (VMMR0IsLogFlushDisabled(pVCpu)) \
7932 HMVMX_ASSERT_PREEMPT_CPUID(); \
7933 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7934 } while (0)
7935
7936# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7937 do { \
7938 Log4Func(("\n")); \
7939 } while(0)
7940#else /* Release builds */
7941# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7942# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7943#endif
7944
7945
7946/**
7947 * Advances the guest RIP after reading it from the VMCS.
7948 *
7949 * @returns VBox status code.
7950 * @param pVCpu Pointer to the VMCPU.
7951 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7952 * out-of-sync. Make sure to update the required fields
7953 * before using them.
7954 * @param pVmxTransient Pointer to the VMX transient structure.
7955 *
7956 * @remarks No-long-jump zone!!!
7957 */
7958DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7959{
7960 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7961 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7962 AssertRCReturn(rc, rc);
7963
7964 pMixedCtx->rip += pVmxTransient->cbInstr;
7965 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
7966 return rc;
7967}
7968
7969
7970/**
7971 * Tries to determine what part of the guest-state VT-x has deemed as invalid
7972 * and update error record fields accordingly.
7973 *
7974 * @return VMX_IGS_* return codes.
7975 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7976 * wrong with the guest state.
7977 *
7978 * @param pVM Pointer to the VM.
7979 * @param pVCpu Pointer to the VMCPU.
7980 * @param pCtx Pointer to the guest-CPU state.
7981 */
7982static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7983{
7984#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7985#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
7986 uError = (err); \
7987 break; \
7988 } else do {} while (0)
7989/* Duplicate of IEM_IS_CANONICAL(). */
7990#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
7991
7992 int rc;
7993 uint64_t u64Val;
7994 uint32_t u32Val;
7995 uint32_t uError = VMX_IGS_ERROR;
7996 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
7997
7998 do
7999 {
8000 /*
8001 * CR0.
8002 */
8003 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8004 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8005 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8006 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8007 if (fUnrestrictedGuest)
8008 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8009
8010 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
8011 AssertRCBreak(rc);
8012 HMVMX_CHECK_BREAK((u32Val & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8013 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8014 if ( !fUnrestrictedGuest
8015 && (u32Val & X86_CR0_PG)
8016 && !(u32Val & X86_CR0_PE))
8017 {
8018 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8019 }
8020
8021 /*
8022 * CR4.
8023 */
8024 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8025 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8026 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
8027 AssertRCBreak(rc);
8028 HMVMX_CHECK_BREAK((u32Val & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8029 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8030
8031 /*
8032 * IA32_DEBUGCTL MSR.
8033 */
8034 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8035 AssertRCBreak(rc);
8036 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8037 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8038 {
8039 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8040 }
8041 uint64_t u64DebugCtlMsr = u64Val;
8042
8043#ifdef VBOX_STRICT
8044 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
8045 AssertRCBreak(rc);
8046 Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8047#endif
8048 bool const fLongModeGuest = !!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8049
8050 /*
8051 * RIP and RFLAGS.
8052 */
8053 uint32_t u32Eflags;
8054#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8055 if (HMVMX_IS_64BIT_HOST_MODE())
8056 {
8057 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8058 AssertRCBreak(rc);
8059 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8060 if ( !fLongModeGuest
8061 || !pCtx->cs.Attr.n.u1Long)
8062 {
8063 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8064 }
8065 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8066 * must be identical if the "IA32e mode guest" VM-entry control is 1
8067 * and CS.L is 1. No check applies if the CPU supports 64
8068 * linear-address bits. */
8069
8070 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8071 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8072 AssertRCBreak(rc);
8073 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8074 VMX_IGS_RFLAGS_RESERVED);
8075 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8076 u32Eflags = u64Val;
8077 }
8078 else
8079#endif
8080 {
8081 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8082 AssertRCBreak(rc);
8083 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8084 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8085 }
8086
8087 if ( fLongModeGuest
8088 || !(pCtx->cr0 & X86_CR0_PE))
8089 {
8090 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8091 }
8092
8093 uint32_t u32EntryInfo;
8094 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8095 AssertRCBreak(rc);
8096 if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8097 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8098 {
8099 HMVMX_CHECK_BREAK(u32Val & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8100 }
8101
8102 /*
8103 * 64-bit checks.
8104 */
8105#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8106 if (HMVMX_IS_64BIT_HOST_MODE())
8107 {
8108 if ( fLongModeGuest
8109 && !fUnrestrictedGuest)
8110 {
8111 HMVMX_CHECK_BREAK(CPUMIsGuestPagingEnabledEx(pCtx), VMX_IGS_CR0_PG_LONGMODE);
8112 HMVMX_CHECK_BREAK((pCtx->cr4 & X86_CR4_PAE), VMX_IGS_CR4_PAE_LONGMODE);
8113 }
8114
8115 if ( !fLongModeGuest
8116 && (pCtx->cr4 & X86_CR4_PCIDE))
8117 {
8118 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8119 }
8120
8121 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8122 * 51:32 beyond the processor's physical-address width are 0. */
8123
8124 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8125 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8126 {
8127 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8128 }
8129
8130 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8131 AssertRCBreak(rc);
8132 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8133
8134 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8135 AssertRCBreak(rc);
8136 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8137 }
8138#endif
8139
8140 /*
8141 * PERF_GLOBAL MSR.
8142 */
8143 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8144 {
8145 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8146 AssertRCBreak(rc);
8147 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8148 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8149 }
8150
8151 /*
8152 * PAT MSR.
8153 */
8154 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8155 {
8156 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8157 AssertRCBreak(rc);
8158 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8159 for (unsigned i = 0; i < 8; i++)
8160 {
8161 uint8_t u8Val = (u64Val & 0x7);
8162 if ( u8Val != 0 /* UC */
8163 || u8Val != 1 /* WC */
8164 || u8Val != 4 /* WT */
8165 || u8Val != 5 /* WP */
8166 || u8Val != 6 /* WB */
8167 || u8Val != 7 /* UC- */)
8168 {
8169 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8170 }
8171 u64Val >>= 3;
8172 }
8173 }
8174
8175 /*
8176 * EFER MSR.
8177 */
8178 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8179 {
8180 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8181 AssertRCBreak(rc);
8182 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8183 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8184 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8185 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8186 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8187 || (u64Val & MSR_K6_EFER_LMA) == (pCtx->cr0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8188 }
8189
8190 /*
8191 * Segment registers.
8192 */
8193 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8194 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8195 if (!(u32Eflags & X86_EFL_VM))
8196 {
8197 /* CS */
8198 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8199 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8200 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8201 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8202 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8203 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8204 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8205 /* CS cannot be loaded with NULL in protected mode. */
8206 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8207 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8208 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8209 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8210 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8211 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8212 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8213 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8214 else
8215 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8216
8217 /* SS */
8218 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8219 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8220 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8221 if ( !(pCtx->cr0 & X86_CR0_PE)
8222 || pCtx->cs.Attr.n.u4Type == 3)
8223 {
8224 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8225 }
8226 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8227 {
8228 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8229 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8230 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8231 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8232 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8233 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8234 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8235 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8236 }
8237
8238 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8239 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8240 {
8241 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8242 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8243 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8244 || pCtx->ds.Attr.n.u4Type > 11
8245 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8246 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8247 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8248 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8249 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8250 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8251 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8252 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8253 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8254 }
8255 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8256 {
8257 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8258 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8259 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8260 || pCtx->es.Attr.n.u4Type > 11
8261 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8262 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8263 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8264 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8265 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8266 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8267 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8268 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8269 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8270 }
8271 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8272 {
8273 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8274 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8275 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8276 || pCtx->fs.Attr.n.u4Type > 11
8277 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8278 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8279 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8280 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8281 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8282 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8283 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8284 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8285 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8286 }
8287 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8288 {
8289 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8290 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8291 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8292 || pCtx->gs.Attr.n.u4Type > 11
8293 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8294 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8295 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8296 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8297 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8298 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8299 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8300 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8301 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8302 }
8303 /* 64-bit capable CPUs. */
8304#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8305 if (HMVMX_IS_64BIT_HOST_MODE())
8306 {
8307 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8308 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8309 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8310 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8311 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8312 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8313 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8314 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8315 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8316 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8317 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8318 }
8319#endif
8320 }
8321 else
8322 {
8323 /* V86 mode checks. */
8324 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8325 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8326 {
8327 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8328 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8329 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8330 }
8331 else
8332 {
8333 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8334 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8335 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8336 }
8337
8338 /* CS */
8339 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8340 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8341 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8342 /* SS */
8343 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8344 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8345 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8346 /* DS */
8347 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8348 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8349 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8350 /* ES */
8351 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8352 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8353 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8354 /* FS */
8355 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8356 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8357 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8358 /* GS */
8359 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8360 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8361 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8362 /* 64-bit capable CPUs. */
8363#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8364 if (HMVMX_IS_64BIT_HOST_MODE())
8365 {
8366 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8367 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8368 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8369 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8370 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8371 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8372 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8373 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8374 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8375 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8376 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8377 }
8378#endif
8379 }
8380
8381 /*
8382 * TR.
8383 */
8384 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8385 /* 64-bit capable CPUs. */
8386#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8387 if (HMVMX_IS_64BIT_HOST_MODE())
8388 {
8389 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8390 }
8391#endif
8392 if (fLongModeGuest)
8393 {
8394 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8395 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8396 }
8397 else
8398 {
8399 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8400 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8401 VMX_IGS_TR_ATTR_TYPE_INVALID);
8402 }
8403 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8404 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8405 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8406 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8407 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8408 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8409 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8410 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8411
8412 /*
8413 * GDTR and IDTR.
8414 */
8415#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8416 if (HMVMX_IS_64BIT_HOST_MODE())
8417 {
8418 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8419 AssertRCBreak(rc);
8420 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8421
8422 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8423 AssertRCBreak(rc);
8424 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8425 }
8426#endif
8427
8428 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8429 AssertRCBreak(rc);
8430 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8431
8432 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8433 AssertRCBreak(rc);
8434 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8435
8436 /*
8437 * Guest Non-Register State.
8438 */
8439 /* Activity State. */
8440 uint32_t u32ActivityState;
8441 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8442 AssertRCBreak(rc);
8443 HMVMX_CHECK_BREAK( !u32ActivityState
8444 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8445 VMX_IGS_ACTIVITY_STATE_INVALID);
8446 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8447 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8448 uint32_t u32IntrState;
8449 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8450 AssertRCBreak(rc);
8451 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8452 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8453 {
8454 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8455 }
8456
8457 /** @todo Activity state and injecting interrupts. Left as a todo since we
8458 * currently don't use activity states but ACTIVE. */
8459
8460 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8461 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8462
8463 /* Guest interruptibility-state. */
8464 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8465 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8466 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8467 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8468 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8469 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8470 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8471 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8472 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8473 if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8474 {
8475 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8476 {
8477 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8478 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8479 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8480 }
8481 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8482 {
8483 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8484 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8485 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8486 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8487 }
8488 }
8489 /** @todo Assumes the processor is not in SMM. */
8490 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8491 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8492 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8493 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8494 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8495 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8496 && VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8497 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8498 {
8499 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8500 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8501 }
8502
8503 /* Pending debug exceptions. */
8504 if (HMVMX_IS_64BIT_HOST_MODE())
8505 {
8506 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8507 AssertRCBreak(rc);
8508 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8509 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8510 u32Val = u64Val; /* For pending debug exceptions checks below. */
8511 }
8512 else
8513 {
8514 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8515 AssertRCBreak(rc);
8516 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8517 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8518 }
8519
8520 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8521 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8522 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8523 {
8524 if ( (u32Eflags & X86_EFL_TF)
8525 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8526 {
8527 /* Bit 14 is PendingDebug.BS. */
8528 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8529 }
8530 if ( !(u32Eflags & X86_EFL_TF)
8531 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8532 {
8533 /* Bit 14 is PendingDebug.BS. */
8534 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8535 }
8536 }
8537
8538 /* VMCS link pointer. */
8539 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8540 AssertRCBreak(rc);
8541 if (u64Val != UINT64_C(0xffffffffffffffff))
8542 {
8543 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8544 /** @todo Bits beyond the processor's physical-address width MBZ. */
8545 /** @todo 32-bit located in memory referenced by value of this field (as a
8546 * physical address) must contain the processor's VMCS revision ID. */
8547 /** @todo SMM checks. */
8548 }
8549
8550 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8551
8552 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8553 if (uError == VMX_IGS_ERROR)
8554 uError = VMX_IGS_REASON_NOT_FOUND;
8555 } while (0);
8556
8557 pVCpu->hm.s.u32HMError = uError;
8558 return uError;
8559
8560#undef HMVMX_ERROR_BREAK
8561#undef HMVMX_CHECK_BREAK
8562#undef HMVMX_IS_CANONICAL
8563}
8564
8565/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8566/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8567/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8568
8569/** @name VM-exit handlers.
8570 * @{
8571 */
8572
8573/**
8574 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8575 */
8576HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8577{
8578 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8579 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8580 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8581#if HC_ARCH_BITS == 64
8582 Assert(ASMIntAreEnabled());
8583 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8584 return VINF_SUCCESS;
8585#endif
8586 return VINF_EM_RAW_INTERRUPT;
8587}
8588
8589
8590/**
8591 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8592 */
8593HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8594{
8595 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8596 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8597
8598 int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8599 AssertRCReturn(rc, rc);
8600
8601 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8602 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8603 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8604 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8605
8606 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8607 {
8608 /*
8609 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8610 * anything we inject is not going to cause a VM-exit directly for the event being injected.
8611 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8612 *
8613 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8614 */
8615 VMXDispatchHostNmi();
8616 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
8617 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8618 return VINF_SUCCESS;
8619 }
8620
8621 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8622 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8623 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8624 {
8625 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8626 return VINF_SUCCESS;
8627 }
8628 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8629 {
8630 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8631 return rc;
8632 }
8633
8634 uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8635 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8636 switch (uIntrType)
8637 {
8638 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8639 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8640 /* no break */
8641 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8642 {
8643 switch (uVector)
8644 {
8645 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8646 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8647 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8648 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8649 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8650 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8651#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8652 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8653 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8654 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8655 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8656 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8657 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8658 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8659 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8660 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8661 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8662#endif
8663 default:
8664 {
8665 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8666 AssertRCReturn(rc, rc);
8667
8668 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8669 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8670 {
8671 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8672 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8673 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8674
8675 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8676 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8677 AssertRCReturn(rc, rc);
8678 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8679 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8680 0 /* GCPtrFaultAddress */);
8681 AssertRCReturn(rc, rc);
8682 }
8683 else
8684 {
8685 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8686 pVCpu->hm.s.u32HMError = uVector;
8687 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8688 }
8689 break;
8690 }
8691 }
8692 break;
8693 }
8694
8695 default:
8696 {
8697 pVCpu->hm.s.u32HMError = uExitIntrInfo;
8698 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8699 AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8700 break;
8701 }
8702 }
8703 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8704 return rc;
8705}
8706
8707
8708/**
8709 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8710 */
8711HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8712{
8713 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8714
8715 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8716 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8717 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8718 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8719 AssertRCReturn(rc, rc);
8720
8721 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8722 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8723 return VINF_SUCCESS;
8724}
8725
8726
8727/**
8728 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8729 */
8730HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8731{
8732 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8733 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8734 pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8735 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8736}
8737
8738
8739/**
8740 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8741 */
8742HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8743{
8744 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8745 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8746 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8747}
8748
8749
8750/**
8751 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8752 */
8753HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8754{
8755 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8756 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8757 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8758}
8759
8760
8761/**
8762 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8763 */
8764HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8765{
8766 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8767 PVM pVM = pVCpu->CTX_SUFF(pVM);
8768 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8769 if (RT_LIKELY(rc == VINF_SUCCESS))
8770 {
8771 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8772 Assert(pVmxTransient->cbInstr == 2);
8773 }
8774 else
8775 {
8776 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8777 rc = VERR_EM_INTERPRETER;
8778 }
8779 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8780 return rc;
8781}
8782
8783
8784/**
8785 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8786 */
8787HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8788{
8789 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8790 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8791 AssertRCReturn(rc, rc);
8792
8793 if (pMixedCtx->cr4 & X86_CR4_SMXE)
8794 return VINF_EM_RAW_EMULATE_INSTR;
8795
8796 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8797 pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8798 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8799}
8800
8801
8802/**
8803 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8804 */
8805HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8806{
8807 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8808 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8809 AssertRCReturn(rc, rc);
8810
8811 PVM pVM = pVCpu->CTX_SUFF(pVM);
8812 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8813 if (RT_LIKELY(rc == VINF_SUCCESS))
8814 {
8815 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8816 Assert(pVmxTransient->cbInstr == 2);
8817 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8818 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8819 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8820 }
8821 else
8822 {
8823 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8824 rc = VERR_EM_INTERPRETER;
8825 }
8826 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8827 return rc;
8828}
8829
8830
8831/**
8832 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8833 */
8834HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8835{
8836 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8837 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8838 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8839 AssertRCReturn(rc, rc);
8840
8841 PVM pVM = pVCpu->CTX_SUFF(pVM);
8842 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8843 if (RT_LIKELY(rc == VINF_SUCCESS))
8844 {
8845 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8846 Assert(pVmxTransient->cbInstr == 3);
8847 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8848 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8849 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8850 }
8851 else
8852 {
8853 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8854 rc = VERR_EM_INTERPRETER;
8855 }
8856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8857 return rc;
8858}
8859
8860
8861/**
8862 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8863 */
8864HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8865{
8866 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8867 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8868 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8869 AssertRCReturn(rc, rc);
8870
8871 PVM pVM = pVCpu->CTX_SUFF(pVM);
8872 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8873 if (RT_LIKELY(rc == VINF_SUCCESS))
8874 {
8875 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8876 Assert(pVmxTransient->cbInstr == 2);
8877 }
8878 else
8879 {
8880 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8881 rc = VERR_EM_INTERPRETER;
8882 }
8883 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8884 return rc;
8885}
8886
8887
8888/**
8889 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8890 */
8891HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8892{
8893 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8894 PVM pVM = pVCpu->CTX_SUFF(pVM);
8895 Assert(!pVM->hm.s.fNestedPaging);
8896
8897 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8898 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8899 AssertRCReturn(rc, rc);
8900
8901 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8902 rc = VBOXSTRICTRC_VAL(rc2);
8903 if (RT_LIKELY(rc == VINF_SUCCESS))
8904 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8905 else
8906 {
8907 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8908 pVmxTransient->uExitQualification, rc));
8909 }
8910 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8911 return rc;
8912}
8913
8914
8915/**
8916 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8917 */
8918HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8919{
8920 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8921 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8922 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8923 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8924 AssertRCReturn(rc, rc);
8925
8926 PVM pVM = pVCpu->CTX_SUFF(pVM);
8927 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8928 if (RT_LIKELY(rc == VINF_SUCCESS))
8929 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8930 else
8931 {
8932 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8933 rc = VERR_EM_INTERPRETER;
8934 }
8935 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8936 return rc;
8937}
8938
8939
8940/**
8941 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8942 */
8943HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8944{
8945 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8946 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8947 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8948 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8949 AssertRCReturn(rc, rc);
8950
8951 PVM pVM = pVCpu->CTX_SUFF(pVM);
8952 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8953 rc = VBOXSTRICTRC_VAL(rc2);
8954 if (RT_LIKELY( rc == VINF_SUCCESS
8955 || rc == VINF_EM_HALT))
8956 {
8957 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8958 AssertRCReturn(rc3, rc3);
8959
8960 if ( rc == VINF_EM_HALT
8961 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
8962 {
8963 rc = VINF_SUCCESS;
8964 }
8965 }
8966 else
8967 {
8968 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
8969 rc = VERR_EM_INTERPRETER;
8970 }
8971 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
8972 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
8973 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
8974 return rc;
8975}
8976
8977
8978/**
8979 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
8980 */
8981HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8982{
8983 /*
8984 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
8985 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
8986 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
8987 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
8988 */
8989 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8990 pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
8991 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8992}
8993
8994
8995/**
8996 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
8997 */
8998HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8999{
9000 /*
9001 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
9002 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
9003 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
9004 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
9005 */
9006 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9007 pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
9008 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9009}
9010
9011
9012/**
9013 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9014 */
9015HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9016{
9017 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9018 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9019 pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
9020 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9021}
9022
9023
9024/**
9025 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9026 */
9027HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9028{
9029 /*
9030 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9031 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9032 * See Intel spec. 25.3 "Other Causes of VM-exits".
9033 */
9034 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9035 pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
9036 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9037}
9038
9039
9040/**
9041 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9042 * VM-exit.
9043 */
9044HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9045{
9046 /*
9047 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9048 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9049 *
9050 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9051 * See Intel spec. "23.8 Restrictions on VMX operation".
9052 */
9053 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9054 return VINF_SUCCESS;
9055}
9056
9057
9058/**
9059 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9060 * VM-exit.
9061 */
9062HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9063{
9064 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9065 return VINF_EM_RESET;
9066}
9067
9068
9069/**
9070 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9071 */
9072HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9073{
9074 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9075 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9076 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9077 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9078 AssertRCReturn(rc, rc);
9079
9080 pMixedCtx->rip++;
9081 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9082 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9083 rc = VINF_SUCCESS;
9084 else
9085 rc = VINF_EM_HALT;
9086
9087 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9088 return rc;
9089}
9090
9091
9092/**
9093 * VM-exit handler for instructions that result in a #UD exception delivered to
9094 * the guest.
9095 */
9096HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9097{
9098 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9099 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9100 return VINF_SUCCESS;
9101}
9102
9103
9104/**
9105 * VM-exit handler for expiry of the VMX preemption timer.
9106 */
9107HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9108{
9109 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9110
9111 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9112 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9113
9114 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9115 PVM pVM = pVCpu->CTX_SUFF(pVM);
9116 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9117 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9118 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9119}
9120
9121
9122/**
9123 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9124 */
9125HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9126{
9127 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9128
9129 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9130 /** @todo check if XSETBV is supported by the recompiler. */
9131 return VERR_EM_INTERPRETER;
9132}
9133
9134
9135/**
9136 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9137 */
9138HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9139{
9140 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9141
9142 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9143 /** @todo implement EMInterpretInvpcid() */
9144 return VERR_EM_INTERPRETER;
9145}
9146
9147
9148/**
9149 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9150 * Error VM-exit.
9151 */
9152HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9153{
9154 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9155 AssertRCReturn(rc, rc);
9156
9157 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9158 NOREF(uInvalidReason);
9159
9160#ifdef VBOX_STRICT
9161 uint32_t uIntrState;
9162 HMVMXHCUINTREG uHCReg;
9163 uint64_t u64Val;
9164 uint32_t u32Val;
9165
9166 rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
9167 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9168 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9169 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9170 AssertRCReturn(rc, rc);
9171
9172 Log4(("uInvalidReason %u\n", uInvalidReason));
9173 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
9174 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9175 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9176 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9177
9178 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9179 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9180 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9181 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9182 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9183 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9184 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9185 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9186 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9187 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9188 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9189 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9190#endif
9191
9192 PVM pVM = pVCpu->CTX_SUFF(pVM);
9193 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9194
9195 return VERR_VMX_INVALID_GUEST_STATE;
9196}
9197
9198
9199/**
9200 * VM-exit handler for VM-entry failure due to an MSR-load
9201 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9202 */
9203HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9204{
9205 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9206 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9207}
9208
9209
9210/**
9211 * VM-exit handler for VM-entry failure due to a machine-check event
9212 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9213 */
9214HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9215{
9216 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9217 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9218}
9219
9220
9221/**
9222 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9223 * theory.
9224 */
9225HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9226{
9227 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9228 return VERR_VMX_UNDEFINED_EXIT_CODE;
9229}
9230
9231
9232/**
9233 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9234 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9235 * Conditional VM-exit.
9236 */
9237HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9238{
9239 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9240
9241 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9242 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9243 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9244 return VERR_EM_INTERPRETER;
9245 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9246 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9247}
9248
9249
9250/**
9251 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9252 */
9253HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9254{
9255 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9256
9257 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9258 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9259 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9260 return VERR_EM_INTERPRETER;
9261 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9262 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9263}
9264
9265
9266/**
9267 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9268 */
9269HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9270{
9271 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9272
9273 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9274 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9275 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9276 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9277 AssertRCReturn(rc, rc);
9278 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9279
9280 PVM pVM = pVCpu->CTX_SUFF(pVM);
9281 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9282 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9283 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9284 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9285
9286 if (RT_LIKELY(rc == VINF_SUCCESS))
9287 {
9288 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9289 Assert(pVmxTransient->cbInstr == 2);
9290 }
9291 return rc;
9292}
9293
9294
9295/**
9296 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9297 */
9298HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9299{
9300 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9301 PVM pVM = pVCpu->CTX_SUFF(pVM);
9302 int rc = VINF_SUCCESS;
9303
9304 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9305 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9306 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9307 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9308 AssertRCReturn(rc, rc);
9309 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9310
9311 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9312 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9313 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9314
9315 if (RT_LIKELY(rc == VINF_SUCCESS))
9316 {
9317 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9318
9319 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9320 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9321 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9322 {
9323 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9324 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9325 EMInterpretWrmsr() changes it. */
9326 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9327 }
9328 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9329 {
9330 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9331 AssertRCReturn(rc, rc);
9332 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
9333 }
9334 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9335 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9336
9337 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9338 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9339 {
9340 switch (pMixedCtx->ecx)
9341 {
9342 case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
9343 case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
9344 case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
9345 case MSR_K8_FS_BASE: /* no break */
9346 case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS; break;
9347 case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
9348 }
9349 }
9350#ifdef VBOX_STRICT
9351 else
9352 {
9353 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9354 switch (pMixedCtx->ecx)
9355 {
9356 case MSR_IA32_SYSENTER_CS:
9357 case MSR_IA32_SYSENTER_EIP:
9358 case MSR_IA32_SYSENTER_ESP:
9359 case MSR_K8_FS_BASE:
9360 case MSR_K8_GS_BASE:
9361 {
9362 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9363 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9364 }
9365
9366 case MSR_K8_LSTAR:
9367 case MSR_K6_STAR:
9368 case MSR_K8_SF_MASK:
9369 case MSR_K8_TSC_AUX:
9370 case MSR_K8_KERNEL_GS_BASE:
9371 {
9372 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9373 pMixedCtx->ecx));
9374 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9375 }
9376 }
9377 }
9378#endif /* VBOX_STRICT */
9379 }
9380 return rc;
9381}
9382
9383
9384/**
9385 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9386 */
9387HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9388{
9389 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9390
9391 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9392 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9393 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9394 return VERR_EM_INTERPRETER;
9395 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9396 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9397}
9398
9399
9400/**
9401 * VM-exit handler for when the TPR value is lowered below the specified
9402 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9403 */
9404HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9405{
9406 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9407 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9408
9409 /*
9410 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9411 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9412 * resume guest execution.
9413 */
9414 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9415 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9416 return VINF_SUCCESS;
9417}
9418
9419
9420/**
9421 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9422 * VM-exit.
9423 *
9424 * @retval VINF_SUCCESS when guest execution can continue.
9425 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9426 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9427 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9428 * recompiler.
9429 */
9430HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9431{
9432 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9433 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9434 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9435 AssertRCReturn(rc, rc);
9436
9437 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9438 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9439 PVM pVM = pVCpu->CTX_SUFF(pVM);
9440 switch (uAccessType)
9441 {
9442 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9443 {
9444#if 0
9445 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9446 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9447#else
9448 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9449 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9450 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9451#endif
9452 AssertRCReturn(rc, rc);
9453
9454 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9455 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9456 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9457 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9458
9459 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9460 {
9461 case 0: /* CR0 */
9462 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9463 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9464 break;
9465 case 2: /* C2 **/
9466 /* Nothing to do here, CR2 it's not part of the VMCS. */
9467 break;
9468 case 3: /* CR3 */
9469 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9470 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9471 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR3;
9472 break;
9473 case 4: /* CR4 */
9474 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9475 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR4;
9476 break;
9477 case 8: /* CR8 */
9478 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9479 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9480 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9481 break;
9482 default:
9483 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9484 break;
9485 }
9486
9487 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9488 break;
9489 }
9490
9491 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9492 {
9493 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9494 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9495 AssertRCReturn(rc, rc);
9496 Assert( !pVM->hm.s.fNestedPaging
9497 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9498 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9499
9500 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9501 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9502 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9503
9504 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9505 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9506 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9507 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9508 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9509 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9510 break;
9511 }
9512
9513 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9514 {
9515 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9516 AssertRCReturn(rc, rc);
9517 rc = EMInterpretCLTS(pVM, pVCpu);
9518 AssertRCReturn(rc, rc);
9519 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9520 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9521 Log4(("CRX CLTS write rc=%d\n", rc));
9522 break;
9523 }
9524
9525 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9526 {
9527 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9528 AssertRCReturn(rc, rc);
9529 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9530 if (RT_LIKELY(rc == VINF_SUCCESS))
9531 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9532 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9533 Log4(("CRX LMSW write rc=%d\n", rc));
9534 break;
9535 }
9536
9537 default:
9538 {
9539 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9540 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9541 }
9542 }
9543
9544 /* Validate possible error codes. */
9545 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9546 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9547 if (RT_SUCCESS(rc))
9548 {
9549 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9550 AssertRCReturn(rc2, rc2);
9551 }
9552
9553 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9554 return rc;
9555}
9556
9557
9558/**
9559 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9560 * VM-exit.
9561 */
9562HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9563{
9564 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9565 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9566
9567 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9568 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9569 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9570 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9571 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9572 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9573 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9574 AssertRCReturn(rc2, rc2);
9575
9576 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9577 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9578 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9579 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9580 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9581 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9582 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9583
9584 /* I/O operation lookup arrays. */
9585 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9586 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9587
9588 VBOXSTRICTRC rcStrict;
9589 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9590 const uint32_t cbInstr = pVmxTransient->cbInstr;
9591 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9592 PVM pVM = pVCpu->CTX_SUFF(pVM);
9593 if (fIOString)
9594 {
9595 /*
9596 * INS/OUTS - I/O String instruction.
9597 *
9598 * Use instruction-information if available, otherwise fall back on
9599 * interpreting the instruction.
9600 */
9601 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9602#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9603 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9604 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9605 {
9606 rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9607 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9608 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9609 AssertRCReturn(rc2, rc2);
9610 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9611 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9612 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9613 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9614 if (fIOWrite)
9615 {
9616 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9617 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9618 //if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9619 // hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9620 // pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9621 }
9622 else
9623 {
9624 AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9625 ("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9626 VERR_HMVMX_IPE_4);
9627 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9628 //if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9629 // hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9630 }
9631 }
9632 else
9633 {
9634 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9635 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9636 AssertRCReturn(rc2, rc2);
9637 rcStrict = IEMExecOne(pVCpu);
9638 }
9639 /** @todo IEM needs to be setting these flags somehow. */
9640 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9641 fUpdateRipAlready = true;
9642#else
9643 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9644 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9645 if (RT_SUCCESS(rcStrict))
9646 {
9647 if (fIOWrite)
9648 {
9649 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9650 (DISCPUMODE)pDis->uAddrMode, cbValue);
9651 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9652 }
9653 else
9654 {
9655 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9656 (DISCPUMODE)pDis->uAddrMode, cbValue);
9657 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9658 }
9659 }
9660 else
9661 {
9662 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9663 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9664 }
9665#endif
9666 }
9667 else
9668 {
9669 /*
9670 * IN/OUT - I/O instruction.
9671 */
9672 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9673 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9674 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9675 if (fIOWrite)
9676 {
9677 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9678 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9679 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9680 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9681 }
9682 else
9683 {
9684 uint32_t u32Result = 0;
9685 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9686 if (IOM_SUCCESS(rcStrict))
9687 {
9688 /* Save result of I/O IN instr. in AL/AX/EAX. */
9689 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
9690 }
9691 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9692 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9693 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9694 }
9695 }
9696
9697 if (IOM_SUCCESS(rcStrict))
9698 {
9699 if (!fUpdateRipAlready)
9700 {
9701 pMixedCtx->rip += cbInstr;
9702 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9703 }
9704
9705 /*
9706 * If any I/O breakpoints are armed, we need to check if one triggered
9707 * and take appropriate action.
9708 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9709 */
9710 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9711 AssertRCReturn(rc2, rc2);
9712
9713 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9714 * execution engines about whether hyper BPs and such are pending. */
9715 uint32_t const uDr7 = pMixedCtx->dr[7];
9716 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9717 && X86_DR7_ANY_RW_IO(uDr7)
9718 && (pMixedCtx->cr4 & X86_CR4_DE))
9719 || DBGFBpIsHwIoArmed(pVM)))
9720 {
9721 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9722
9723 /* We're playing with the host CPU state here, make sure we don't preempt. */
9724 HM_DISABLE_PREEMPT_IF_NEEDED();
9725 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
9726
9727 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9728 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9729 {
9730 /* Raise #DB. */
9731 if (fIsGuestDbgActive)
9732 ASMSetDR6(pMixedCtx->dr[6]);
9733 if (pMixedCtx->dr[7] != uDr7)
9734 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9735
9736 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9737 }
9738 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9739 else if ( rcStrict2 != VINF_SUCCESS
9740 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
9741 rcStrict = rcStrict2;
9742
9743 HM_RESTORE_PREEMPT_IF_NEEDED();
9744 }
9745 }
9746
9747#ifdef DEBUG
9748 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9749 Assert(!fIOWrite);
9750 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9751 Assert(fIOWrite);
9752 else
9753 {
9754 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9755 * statuses, that the VMM device and some others may return. See
9756 * IOM_SUCCESS() for guidance. */
9757 AssertMsg( RT_FAILURE(rcStrict)
9758 || rcStrict == VINF_SUCCESS
9759 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
9760 || rcStrict == VINF_EM_DBG_BREAKPOINT
9761 || rcStrict == VINF_EM_RAW_GUEST_TRAP
9762 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9763 }
9764#endif
9765
9766 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9767 return VBOXSTRICTRC_TODO(rcStrict);
9768}
9769
9770
9771/**
9772 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9773 * VM-exit.
9774 */
9775HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9776{
9777 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9778
9779 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
9780 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9781 AssertRCReturn(rc, rc);
9782 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9783 {
9784 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9785 AssertRCReturn(rc, rc);
9786 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9787 {
9788 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9789
9790 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9791 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9792 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9793 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9794 {
9795 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9796 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9797
9798 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9799 Assert(!pVCpu->hm.s.Event.fPending);
9800 pVCpu->hm.s.Event.fPending = true;
9801 pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9802 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9803 AssertRCReturn(rc, rc);
9804 if (fErrorCodeValid)
9805 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9806 else
9807 pVCpu->hm.s.Event.u32ErrCode = 0;
9808 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9809 && uVector == X86_XCPT_PF)
9810 {
9811 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9812 }
9813
9814 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9815 }
9816 }
9817 }
9818
9819 /** @todo Emulate task switch someday, currently just going back to ring-3 for
9820 * emulation. */
9821 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9822 return VERR_EM_INTERPRETER;
9823}
9824
9825
9826/**
9827 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9828 */
9829HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9830{
9831 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9832 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9833 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9834 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9835 AssertRCReturn(rc, rc);
9836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9837 return VINF_EM_DBG_STEPPED;
9838}
9839
9840
9841/**
9842 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9843 */
9844HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9845{
9846 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9847
9848 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9849 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9850 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9851 return VINF_SUCCESS;
9852 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9853 return rc;
9854
9855#if 0
9856 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9857 * just sync the whole thing. */
9858 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9859#else
9860 /* Aggressive state sync. for now. */
9861 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9862 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9863 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9864#endif
9865 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9866 AssertRCReturn(rc, rc);
9867
9868 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9869 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9870 switch (uAccessType)
9871 {
9872 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9873 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9874 {
9875 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9876 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9877 {
9878 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9879 }
9880
9881 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9882 GCPhys &= PAGE_BASE_GC_MASK;
9883 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9884 PVM pVM = pVCpu->CTX_SUFF(pVM);
9885 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9886 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9887
9888 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9889 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9890 CPUMCTX2CORE(pMixedCtx), GCPhys);
9891 rc = VBOXSTRICTRC_VAL(rc2);
9892 Log4(("ApicAccess rc=%d\n", rc));
9893 if ( rc == VINF_SUCCESS
9894 || rc == VERR_PAGE_TABLE_NOT_PRESENT
9895 || rc == VERR_PAGE_NOT_PRESENT)
9896 {
9897 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
9898 | HM_CHANGED_VMX_GUEST_APIC_STATE;
9899 rc = VINF_SUCCESS;
9900 }
9901 break;
9902 }
9903
9904 default:
9905 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9906 rc = VINF_EM_RAW_EMULATE_INSTR;
9907 break;
9908 }
9909
9910 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9911 return rc;
9912}
9913
9914
9915/**
9916 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9917 * VM-exit.
9918 */
9919HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9920{
9921 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9922
9923 /* We should -not- get this VM-exit if the guest is debugging. */
9924 if (CPUMIsGuestDebugStateActive(pVCpu))
9925 {
9926 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9927 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9928 }
9929
9930 int rc = VERR_INTERNAL_ERROR_5;
9931 if ( !DBGFIsStepping(pVCpu)
9932 && !pVCpu->hm.s.fSingleInstruction
9933 && !CPUMIsHyperDebugStateActive(pVCpu))
9934 {
9935 /* Don't intercept MOV DRx and #DB any more. */
9936 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
9937 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9938 AssertRCReturn(rc, rc);
9939
9940 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9941 {
9942#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9943 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
9944 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9945 AssertRCReturn(rc, rc);
9946#endif
9947 }
9948
9949 /* We're playing with the host CPU state here, make sure we can't preempt. */
9950 HM_DISABLE_PREEMPT_IF_NEEDED();
9951
9952 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9953 PVM pVM = pVCpu->CTX_SUFF(pVM);
9954 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9955 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
9956
9957 HM_RESTORE_PREEMPT_IF_NEEDED();
9958
9959#ifdef VBOX_WITH_STATISTICS
9960 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9961 AssertRCReturn(rc, rc);
9962 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9963 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9964 else
9965 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9966#endif
9967 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
9968 return VINF_SUCCESS;
9969 }
9970
9971 /*
9972 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
9973 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
9974 */
9975 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9976 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9977 AssertRCReturn(rc, rc);
9978 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9979
9980 PVM pVM = pVCpu->CTX_SUFF(pVM);
9981 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9982 {
9983 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9984 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
9985 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
9986 if (RT_SUCCESS(rc))
9987 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9988 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9989 }
9990 else
9991 {
9992 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9993 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
9994 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
9995 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9996 }
9997
9998 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9999 if (RT_SUCCESS(rc))
10000 {
10001 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10002 AssertRCReturn(rc2, rc2);
10003 }
10004 return rc;
10005}
10006
10007
10008/**
10009 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10010 * Conditional VM-exit.
10011 */
10012HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10013{
10014 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10015 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10016
10017 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10018 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10019 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10020 return VINF_SUCCESS;
10021 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10022 return rc;
10023
10024 RTGCPHYS GCPhys = 0;
10025 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10026
10027#if 0
10028 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10029#else
10030 /* Aggressive state sync. for now. */
10031 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10032 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10033 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10034#endif
10035 AssertRCReturn(rc, rc);
10036
10037 /*
10038 * If we succeed, resume guest execution.
10039 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10040 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10041 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10042 * weird case. See @bugref{6043}.
10043 */
10044 PVM pVM = pVCpu->CTX_SUFF(pVM);
10045 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10046 rc = VBOXSTRICTRC_VAL(rc2);
10047 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10048 if ( rc == VINF_SUCCESS
10049 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10050 || rc == VERR_PAGE_NOT_PRESENT)
10051 {
10052 /* Successfully handled MMIO operation. */
10053 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10054 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10055 rc = VINF_SUCCESS;
10056 }
10057 return rc;
10058}
10059
10060
10061/**
10062 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10063 * VM-exit.
10064 */
10065HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10066{
10067 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10068 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10069
10070 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10071 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10072 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10073 return VINF_SUCCESS;
10074 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10075 return rc;
10076
10077 RTGCPHYS GCPhys = 0;
10078 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10079 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10080#if 0
10081 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10082#else
10083 /* Aggressive state sync. for now. */
10084 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10085 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10086 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10087#endif
10088 AssertRCReturn(rc, rc);
10089
10090 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10091 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10092
10093 RTGCUINT uErrorCode = 0;
10094 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10095 uErrorCode |= X86_TRAP_PF_ID;
10096 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10097 uErrorCode |= X86_TRAP_PF_RW;
10098 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10099 uErrorCode |= X86_TRAP_PF_P;
10100
10101 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10102
10103 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10104 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10105
10106 /* Handle the pagefault trap for the nested shadow table. */
10107 PVM pVM = pVCpu->CTX_SUFF(pVM);
10108 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10109 TRPMResetTrap(pVCpu);
10110
10111 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10112 if ( rc == VINF_SUCCESS
10113 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10114 || rc == VERR_PAGE_NOT_PRESENT)
10115 {
10116 /* Successfully synced our nested page tables. */
10117 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10118 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10119 return VINF_SUCCESS;
10120 }
10121
10122 Log4(("EPT return to ring-3 rc=%d\n"));
10123 return rc;
10124}
10125
10126/** @} */
10127
10128/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10129/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10130/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10131
10132/** @name VM-exit exception handlers.
10133 * @{
10134 */
10135
10136/**
10137 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10138 */
10139static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10140{
10141 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10142 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10143
10144 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10145 AssertRCReturn(rc, rc);
10146
10147 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10148 {
10149 /* Old-style FPU error reporting needs some extra work. */
10150 /** @todo don't fall back to the recompiler, but do it manually. */
10151 return VERR_EM_INTERPRETER;
10152 }
10153
10154 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10155 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10156 return rc;
10157}
10158
10159
10160/**
10161 * VM-exit exception handler for #BP (Breakpoint exception).
10162 */
10163static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10164{
10165 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10166 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10167
10168 /** @todo Try optimize this by not saving the entire guest state unless
10169 * really needed. */
10170 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10171 AssertRCReturn(rc, rc);
10172
10173 PVM pVM = pVCpu->CTX_SUFF(pVM);
10174 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10175 if (rc == VINF_EM_RAW_GUEST_TRAP)
10176 {
10177 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10178 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10179 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10180 AssertRCReturn(rc, rc);
10181
10182 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10183 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10184 }
10185
10186 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10187 return rc;
10188}
10189
10190
10191/**
10192 * VM-exit exception handler for #DB (Debug exception).
10193 */
10194static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10195{
10196 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10197 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10198 Log6(("XcptDB\n"));
10199
10200 /*
10201 * Get the DR6-like values from the exit qualification and pass it to DBGF
10202 * for processing.
10203 */
10204 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10205 AssertRCReturn(rc, rc);
10206
10207 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10208 uint64_t uDR6 = X86_DR6_INIT_VAL;
10209 uDR6 |= ( pVmxTransient->uExitQualification
10210 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10211
10212 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10213 if (rc == VINF_EM_RAW_GUEST_TRAP)
10214 {
10215 /*
10216 * The exception was for the guest. Update DR6, DR7.GD and
10217 * IA32_DEBUGCTL.LBR before forwarding it.
10218 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10219 */
10220 HM_DISABLE_PREEMPT_IF_NEEDED();
10221
10222 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10223 pMixedCtx->dr[6] |= uDR6;
10224 if (CPUMIsGuestDebugStateActive(pVCpu))
10225 ASMSetDR6(pMixedCtx->dr[6]);
10226
10227 HM_RESTORE_PREEMPT_IF_NEEDED();
10228
10229 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10230 AssertRCReturn(rc, rc);
10231
10232 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10233 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10234
10235 /* Paranoia. */
10236 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10237 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10238
10239 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10240 AssertRCReturn(rc, rc);
10241
10242 /*
10243 * Raise #DB in the guest.
10244 */
10245 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10246 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10247 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10248 AssertRCReturn(rc, rc);
10249 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10250 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10251 return VINF_SUCCESS;
10252 }
10253
10254 /*
10255 * Not a guest trap, must be a hypervisor related debug event then.
10256 * Update DR6 in case someone is interested in it.
10257 */
10258 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10259 AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
10260 CPUMSetHyperDR6(pVCpu, uDR6);
10261
10262 return rc;
10263}
10264
10265
10266/**
10267 * VM-exit exception handler for #NM (Device-not-available exception: floating
10268 * point exception).
10269 */
10270static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10271{
10272 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10273
10274#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10275 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
10276#endif
10277
10278 /* We require CR0 and EFER. EFER is always up-to-date. */
10279 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10280 AssertRCReturn(rc, rc);
10281
10282 /* We're playing with the host CPU state here, have to disable preemption. */
10283 HM_DISABLE_PREEMPT_IF_NEEDED();
10284
10285 /* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10286 PVM pVM = pVCpu->CTX_SUFF(pVM);
10287 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
10288 if (rc == VINF_SUCCESS)
10289 {
10290 Assert(CPUMIsGuestFPUStateActive(pVCpu));
10291 HM_RESTORE_PREEMPT_IF_NEEDED();
10292
10293 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
10294 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10295 return VINF_SUCCESS;
10296 }
10297
10298 HM_RESTORE_PREEMPT_IF_NEEDED();
10299
10300 /* Forward #NM to the guest. */
10301 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10302 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10303 AssertRCReturn(rc, rc);
10304 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10305 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10306 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10307 return rc;
10308}
10309
10310
10311/**
10312 * VM-exit exception handler for #GP (General-protection exception).
10313 *
10314 * @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
10315 */
10316static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10317{
10318 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10319 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10320
10321 int rc = VERR_INTERNAL_ERROR_5;
10322 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10323 {
10324#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10325 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10326 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10327 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10328 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10329 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10330 AssertRCReturn(rc, rc);
10331 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10332 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10333 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10334 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10335 return rc;
10336#else
10337 /* We don't intercept #GP. */
10338 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10339 return VERR_VMX_UNEXPECTED_EXCEPTION;
10340#endif
10341 }
10342
10343 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10344 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10345
10346 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10347 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10348 AssertRCReturn(rc, rc);
10349
10350 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10351 uint32_t cbOp = 0;
10352 PVM pVM = pVCpu->CTX_SUFF(pVM);
10353 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10354 if (RT_SUCCESS(rc))
10355 {
10356 rc = VINF_SUCCESS;
10357 Assert(cbOp == pDis->cbInstr);
10358 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10359 switch (pDis->pCurInstr->uOpcode)
10360 {
10361 case OP_CLI:
10362 {
10363 pMixedCtx->eflags.Bits.u1IF = 0;
10364 pMixedCtx->rip += pDis->cbInstr;
10365 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10366 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10367 break;
10368 }
10369
10370 case OP_STI:
10371 {
10372 pMixedCtx->eflags.Bits.u1IF = 1;
10373 pMixedCtx->rip += pDis->cbInstr;
10374 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10375 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10376 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10377 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10378 break;
10379 }
10380
10381 case OP_HLT:
10382 {
10383 rc = VINF_EM_HALT;
10384 pMixedCtx->rip += pDis->cbInstr;
10385 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
10386 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10387 break;
10388 }
10389
10390 case OP_POPF:
10391 {
10392 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10393 uint32_t cbParm = 0;
10394 uint32_t uMask = 0;
10395 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10396 {
10397 cbParm = 4;
10398 uMask = 0xffffffff;
10399 }
10400 else
10401 {
10402 cbParm = 2;
10403 uMask = 0xffff;
10404 }
10405
10406 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10407 RTGCPTR GCPtrStack = 0;
10408 X86EFLAGS Eflags;
10409 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10410 &GCPtrStack);
10411 if (RT_SUCCESS(rc))
10412 {
10413 Assert(sizeof(Eflags.u32) >= cbParm);
10414 Eflags.u32 = 0;
10415 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10416 }
10417 if (RT_FAILURE(rc))
10418 {
10419 rc = VERR_EM_INTERPRETER;
10420 break;
10421 }
10422 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10423 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10424 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10425 /* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10426 pMixedCtx->eflags.Bits.u1RF = 0;
10427 pMixedCtx->esp += cbParm;
10428 pMixedCtx->esp &= uMask;
10429 pMixedCtx->rip += pDis->cbInstr;
10430 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10431 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10432 break;
10433 }
10434
10435 case OP_PUSHF:
10436 {
10437 uint32_t cbParm = 0;
10438 uint32_t uMask = 0;
10439 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10440 {
10441 cbParm = 4;
10442 uMask = 0xffffffff;
10443 }
10444 else
10445 {
10446 cbParm = 2;
10447 uMask = 0xffff;
10448 }
10449
10450 /* Get the stack pointer & push the contents of eflags onto the stack. */
10451 RTGCPTR GCPtrStack = 0;
10452 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10453 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10454 if (RT_FAILURE(rc))
10455 {
10456 rc = VERR_EM_INTERPRETER;
10457 break;
10458 }
10459 X86EFLAGS Eflags = pMixedCtx->eflags;
10460 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10461 Eflags.Bits.u1RF = 0;
10462 Eflags.Bits.u1VM = 0;
10463
10464 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10465 if (RT_FAILURE(rc))
10466 {
10467 rc = VERR_EM_INTERPRETER;
10468 break;
10469 }
10470 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10471 pMixedCtx->esp -= cbParm;
10472 pMixedCtx->esp &= uMask;
10473 pMixedCtx->rip += pDis->cbInstr;
10474 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP;
10475 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10476 break;
10477 }
10478
10479 case OP_IRET:
10480 {
10481 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10482 * instruction reference. */
10483 RTGCPTR GCPtrStack = 0;
10484 uint32_t uMask = 0xffff;
10485 uint16_t aIretFrame[3];
10486 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10487 {
10488 rc = VERR_EM_INTERPRETER;
10489 break;
10490 }
10491 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10492 &GCPtrStack);
10493 if (RT_SUCCESS(rc))
10494 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10495 if (RT_FAILURE(rc))
10496 {
10497 rc = VERR_EM_INTERPRETER;
10498 break;
10499 }
10500 pMixedCtx->eip = 0;
10501 pMixedCtx->ip = aIretFrame[0];
10502 pMixedCtx->cs.Sel = aIretFrame[1];
10503 pMixedCtx->cs.ValidSel = aIretFrame[1];
10504 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10505 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10506 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10507 pMixedCtx->sp += sizeof(aIretFrame);
10508 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_SEGMENT_REGS | HM_CHANGED_GUEST_RSP
10509 | HM_CHANGED_GUEST_RFLAGS;
10510 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10511 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10512 break;
10513 }
10514
10515 case OP_INT:
10516 {
10517 uint16_t uVector = pDis->Param1.uValue & 0xff;
10518 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10519 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10520 break;
10521 }
10522
10523 case OP_INTO:
10524 {
10525 if (pMixedCtx->eflags.Bits.u1OF)
10526 {
10527 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10528 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10529 }
10530 break;
10531 }
10532
10533 default:
10534 {
10535 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10536 EMCODETYPE_SUPERVISOR);
10537 rc = VBOXSTRICTRC_VAL(rc2);
10538 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
10539 Log4(("#GP rc=%Rrc\n", rc));
10540 break;
10541 }
10542 }
10543 }
10544 else
10545 rc = VERR_EM_INTERPRETER;
10546
10547 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10548 ("#GP Unexpected rc=%Rrc\n", rc));
10549 return rc;
10550}
10551
10552
10553/**
10554 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10555 * the exception reported in the VMX transient structure back into the VM.
10556 *
10557 * @remarks Requires uExitIntrInfo in the VMX transient structure to be
10558 * up-to-date.
10559 */
10560static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10561{
10562 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10563
10564 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10565 hmR0VmxCheckExitDueToEventDelivery(). */
10566 int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10567 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10568 AssertRCReturn(rc, rc);
10569 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10570
10571 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10572 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10573 return VINF_SUCCESS;
10574}
10575
10576
10577/**
10578 * VM-exit exception handler for #PF (Page-fault exception).
10579 */
10580static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10581{
10582 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10583 PVM pVM = pVCpu->CTX_SUFF(pVM);
10584 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10585 rc |= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10586 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10587 AssertRCReturn(rc, rc);
10588
10589#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
10590 if (pVM->hm.s.fNestedPaging)
10591 {
10592 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10593 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10594 {
10595 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10596 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10597 0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10598 }
10599 else
10600 {
10601 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10602 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10603 Log4(("Pending #DF due to vectoring #PF. NP\n"));
10604 }
10605 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10606 return rc;
10607 }
10608#else
10609 Assert(!pVM->hm.s.fNestedPaging);
10610#endif
10611
10612 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10613 AssertRCReturn(rc, rc);
10614
10615 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10616 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10617
10618 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10619 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10620 (RTGCPTR)pVmxTransient->uExitQualification);
10621
10622 Log4(("#PF: rc=%Rrc\n", rc));
10623 if (rc == VINF_SUCCESS)
10624 {
10625 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10626 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
10627 * memory? We don't update the whole state here... */
10628 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10629 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10630 TRPMResetTrap(pVCpu);
10631 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10632 return rc;
10633 }
10634 else if (rc == VINF_EM_RAW_GUEST_TRAP)
10635 {
10636 if (!pVmxTransient->fVectoringPF)
10637 {
10638 /* It's a guest page fault and needs to be reflected to the guest. */
10639 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10640 TRPMResetTrap(pVCpu);
10641 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10642 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10643 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10644 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10645 }
10646 else
10647 {
10648 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10649 TRPMResetTrap(pVCpu);
10650 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10651 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10652 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10653 }
10654
10655 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10656 return VINF_SUCCESS;
10657 }
10658
10659 TRPMResetTrap(pVCpu);
10660 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10661 return rc;
10662}
10663
10664/** @} */
10665
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette