VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 48366

Last change on this file since 48366 was 48362, checked in by vboxsync, 11 years ago

VMM/HMVMXR0: Clarify that it's guest-CPU context CR0 and not VMCS CR0.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 446.4 KB
Line 
1/* $Id: HMVMXR0.cpp 48362 2013-09-06 14:40:11Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#endif
45
46
47/*******************************************************************************
48* Defined Constants And Macros *
49*******************************************************************************/
50#if defined(RT_ARCH_AMD64)
51# define HMVMX_IS_64BIT_HOST_MODE() (true)
52typedef RTHCUINTREG HMVMXHCUINTREG;
53#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54extern "C" uint32_t g_fVMXIs64bitHost;
55# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56typedef uint64_t HMVMXHCUINTREG;
57#else
58# define HMVMX_IS_64BIT_HOST_MODE() (false)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#endif
61
62/** Use the function table. */
63#define HMVMX_USE_FUNCTION_TABLE
64
65/** Determine which tagged-TLB flush handler to use. */
66#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71/** @name Updated-guest-state flags.
72 * @{ */
73#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94 | HMVMX_UPDATED_GUEST_RSP \
95 | HMVMX_UPDATED_GUEST_RFLAGS \
96 | HMVMX_UPDATED_GUEST_CR0 \
97 | HMVMX_UPDATED_GUEST_CR3 \
98 | HMVMX_UPDATED_GUEST_CR4 \
99 | HMVMX_UPDATED_GUEST_GDTR \
100 | HMVMX_UPDATED_GUEST_IDTR \
101 | HMVMX_UPDATED_GUEST_LDTR \
102 | HMVMX_UPDATED_GUEST_TR \
103 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104 | HMVMX_UPDATED_GUEST_DEBUG \
105 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112 | HMVMX_UPDATED_GUEST_APIC_STATE)
113/** @} */
114
115/** @name
116 * Flags to skip redundant reads of some common VMCS fields that are not part of
117 * the guest-CPU state but are in the transient structure.
118 */
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually (except #PF). #NM is also
143 * handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
144 * intercepted even in real-mode if we have Nested Paging support.
145 */
146#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
147 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
148 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
149 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
150 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
151 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
152 | RT_BIT(X86_XCPT_XF))
153
154/**
155 * Exception bitmap mask for all contributory exceptions.
156 *
157 * Page fault is deliberately excluded here as it's conditional as to whether
158 * it's contributory or benign. Page faults are handled separately.
159 */
160#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
161 | RT_BIT(X86_XCPT_DE))
162
163/** Maximum VM-instruction error number. */
164#define HMVMX_INSTR_ERROR_MAX 28
165
166/** Profiling macro. */
167#ifdef HM_PROFILE_EXIT_DISPATCH
168# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
169# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
170#else
171# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
173#endif
174
175/** Assert that preemption is disabled or covered by thread-context hooks. */
176#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
177 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
178
179/** Assert that we haven't migrated CPUs when thread-context hooks are not
180 * used. */
181#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
182 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
183 ("Illegal migration! Entered on CPU %u Current %u\n", \
184 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
185
186/*******************************************************************************
187* Structures and Typedefs *
188*******************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG uEflags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
203 uint64_t u64LStarMsr;
204 /** The guest's TPR value used for TPR shadowing. */
205 uint8_t u8GuestTpr;
206 /** Alignment. */
207 uint8_t abAlignment0[7];
208
209 /** The basic VM-exit reason. */
210 uint16_t uExitReason;
211 /** Alignment. */
212 uint16_t u16Alignment0;
213 /** The VM-exit interruption error code. */
214 uint32_t uExitIntrErrorCode;
215 /** The VM-exit exit qualification. */
216 uint64_t uExitQualification;
217
218 /** The VM-exit interruption-information field. */
219 uint32_t uExitIntrInfo;
220 /** The VM-exit instruction-length field. */
221 uint32_t cbInstr;
222 /** The VM-exit instruction-information field. */
223 union
224 {
225 /** Plain unsigned int representation. */
226 uint32_t u;
227 /** INS and OUTS information. */
228 struct
229 {
230 uint32_t u6Reserved0 : 6;
231 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
232 uint32_t u3AddrSize : 3;
233 uint32_t u5Reserved1 : 5;
234 /** The segment register (X86_SREG_XXX). */
235 uint32_t iSegReg : 3;
236 uint32_t uReserved2 : 14;
237 } StrIo;
238 } ExitInstrInfo;
239 /** Whether the VM-entry failed or not. */
240 bool fVMEntryFailed;
241 /** Alignment. */
242 uint8_t abAlignment1[3];
243
244 /** The VM-entry interruption-information field. */
245 uint32_t uEntryIntrInfo;
246 /** The VM-entry exception error code field. */
247 uint32_t uEntryXcptErrorCode;
248 /** The VM-entry instruction length field. */
249 uint32_t cbEntryInstr;
250
251 /** IDT-vectoring information field. */
252 uint32_t uIdtVectoringInfo;
253 /** IDT-vectoring error code. */
254 uint32_t uIdtVectoringErrorCode;
255
256 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
257 uint32_t fVmcsFieldsRead;
258 /** Whether TSC-offsetting should be setup before VM-entry. */
259 bool fUpdateTscOffsettingAndPreemptTimer;
260 /** Whether the VM-exit was caused by a page-fault during delivery of a
261 * contributory exception or a page-fault. */
262 bool fVectoringPF;
263} VMXTRANSIENT;
264AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
265AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
266AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
267AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
268/** Pointer to VMX transient state. */
269typedef VMXTRANSIENT *PVMXTRANSIENT;
270
271
272/**
273 * MSR-bitmap read permissions.
274 */
275typedef enum VMXMSREXITREAD
276{
277 /** Reading this MSR causes a VM-exit. */
278 VMXMSREXIT_INTERCEPT_READ = 0xb,
279 /** Reading this MSR does not cause a VM-exit. */
280 VMXMSREXIT_PASSTHRU_READ
281} VMXMSREXITREAD;
282
283/**
284 * MSR-bitmap write permissions.
285 */
286typedef enum VMXMSREXITWRITE
287{
288 /** Writing to this MSR causes a VM-exit. */
289 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
290 /** Writing to this MSR does not cause a VM-exit. */
291 VMXMSREXIT_PASSTHRU_WRITE
292} VMXMSREXITWRITE;
293
294/**
295 * VMX VM-exit handler.
296 *
297 * @returns VBox status code.
298 * @param pVCpu Pointer to the VMCPU.
299 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
300 * out-of-sync. Make sure to update the required
301 * fields before using them.
302 * @param pVmxTransient Pointer to the VMX-transient structure.
303 */
304#ifndef HMVMX_USE_FUNCTION_TABLE
305typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306#else
307typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308/** Pointer to VM-exit handler. */
309typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
310#endif
311
312
313/*******************************************************************************
314* Internal Functions *
315*******************************************************************************/
316static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
317static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
318static void hmR0VmxClearEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx);
319static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
320 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
321#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
322static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
323#endif
324#ifndef HMVMX_USE_FUNCTION_TABLE
325DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
326# define HMVMX_EXIT_DECL static int
327#else
328# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
329#endif
330
331/** @name VM-exit handlers.
332 * @{
333 */
334static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
335static FNVMXEXITHANDLER hmR0VmxExitExtInt;
336static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
337static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
338static FNVMXEXITHANDLER hmR0VmxExitSipi;
339static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
340static FNVMXEXITHANDLER hmR0VmxExitSmi;
341static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
342static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
343static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
344static FNVMXEXITHANDLER hmR0VmxExitCpuid;
345static FNVMXEXITHANDLER hmR0VmxExitGetsec;
346static FNVMXEXITHANDLER hmR0VmxExitHlt;
347static FNVMXEXITHANDLER hmR0VmxExitInvd;
348static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
349static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
350static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
351static FNVMXEXITHANDLER hmR0VmxExitRsm;
352static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
353static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
354static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
355static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
356static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
357static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
358static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
359static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
360static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
361static FNVMXEXITHANDLER hmR0VmxExitMwait;
362static FNVMXEXITHANDLER hmR0VmxExitMtf;
363static FNVMXEXITHANDLER hmR0VmxExitMonitor;
364static FNVMXEXITHANDLER hmR0VmxExitPause;
365static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
366static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
367static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
368static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
369static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
370static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
371static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
372static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
373static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
374static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
375static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
376static FNVMXEXITHANDLER hmR0VmxExitRdrand;
377static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
378/** @} */
379
380static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
381static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
382static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
383static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
384static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
385static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
386static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
387static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
388
389/*******************************************************************************
390* Global Variables *
391*******************************************************************************/
392#ifdef HMVMX_USE_FUNCTION_TABLE
393
394/**
395 * VMX_EXIT dispatch table.
396 */
397static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
398{
399 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
400 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
401 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
402 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
403 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
404 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
405 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
406 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
407 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
408 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
409 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
410 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
411 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
412 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
413 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
414 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
415 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
416 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
417 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
418 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
419 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
420 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
421 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
422 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
423 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
424 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
425 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
426 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
427 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
428 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
429 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
430 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
431 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
432 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
433 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
434 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
435 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
436 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
437 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
438 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
439 /* 40 UNDEFINED */ hmR0VmxExitPause,
440 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
441 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
442 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
443 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
444 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
445 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
446 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
447 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
448 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
449 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
450 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
451 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
452 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
453 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
454 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
455 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
456 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
457 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
458 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
459};
460#endif /* HMVMX_USE_FUNCTION_TABLE */
461
462#ifdef VBOX_STRICT
463static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
464{
465 /* 0 */ "(Not Used)",
466 /* 1 */ "VMCALL executed in VMX root operation.",
467 /* 2 */ "VMCLEAR with invalid physical address.",
468 /* 3 */ "VMCLEAR with VMXON pointer.",
469 /* 4 */ "VMLAUNCH with non-clear VMCS.",
470 /* 5 */ "VMRESUME with non-launched VMCS.",
471 /* 6 */ "VMRESUME after VMXOFF",
472 /* 7 */ "VM entry with invalid control fields.",
473 /* 8 */ "VM entry with invalid host state fields.",
474 /* 9 */ "VMPTRLD with invalid physical address.",
475 /* 10 */ "VMPTRLD with VMXON pointer.",
476 /* 11 */ "VMPTRLD with incorrect revision identifier.",
477 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
478 /* 13 */ "VMWRITE to read-only VMCS component.",
479 /* 14 */ "(Not Used)",
480 /* 15 */ "VMXON executed in VMX root operation.",
481 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
482 /* 17 */ "VM entry with non-launched executing VMCS.",
483 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
484 /* 19 */ "VMCALL with non-clear VMCS.",
485 /* 20 */ "VMCALL with invalid VM-exit control fields.",
486 /* 21 */ "(Not Used)",
487 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
488 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
489 /* 24 */ "VMCALL with invalid SMM-monitor features.",
490 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
491 /* 26 */ "VM entry with events blocked by MOV SS.",
492 /* 27 */ "(Not Used)",
493 /* 28 */ "Invalid operand to INVEPT/INVVPID."
494};
495#endif /* VBOX_STRICT */
496
497
498
499/**
500 * Updates the VM's last error record. If there was a VMX instruction error,
501 * reads the error data from the VMCS and updates VCPU's last error record as
502 * well.
503 *
504 * @param pVM Pointer to the VM.
505 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
506 * VERR_VMX_UNABLE_TO_START_VM or
507 * VERR_VMX_INVALID_VMCS_FIELD).
508 * @param rc The error code.
509 */
510static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
511{
512 AssertPtr(pVM);
513 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
514 || rc == VERR_VMX_UNABLE_TO_START_VM)
515 {
516 AssertPtrReturnVoid(pVCpu);
517 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
518 }
519 pVM->hm.s.lLastError = rc;
520}
521
522
523/**
524 * Reads the VM-entry interruption-information field from the VMCS into the VMX
525 * transient structure.
526 *
527 * @returns VBox status code.
528 * @param pVmxTransient Pointer to the VMX transient structure.
529 *
530 * @remarks No-long-jump zone!!!
531 */
532DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
533{
534 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
535 AssertRCReturn(rc, rc);
536 return VINF_SUCCESS;
537}
538
539
540/**
541 * Reads the VM-entry exception error code field from the VMCS into
542 * the VMX transient structure.
543 *
544 * @returns VBox status code.
545 * @param pVmxTransient Pointer to the VMX transient structure.
546 *
547 * @remarks No-long-jump zone!!!
548 */
549DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
550{
551 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
552 AssertRCReturn(rc, rc);
553 return VINF_SUCCESS;
554}
555
556
557/**
558 * Reads the VM-entry exception error code field from the VMCS into
559 * the VMX transient structure.
560 *
561 * @returns VBox status code.
562 * @param pVCpu Pointer to the VMCPU.
563 * @param pVmxTransient Pointer to the VMX transient structure.
564 *
565 * @remarks No-long-jump zone!!!
566 */
567DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
568{
569 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
570 AssertRCReturn(rc, rc);
571 return VINF_SUCCESS;
572}
573
574
575/**
576 * Reads the VM-exit interruption-information field from the VMCS into the VMX
577 * transient structure.
578 *
579 * @returns VBox status code.
580 * @param pVCpu Pointer to the VMCPU.
581 * @param pVmxTransient Pointer to the VMX transient structure.
582 */
583DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
584{
585 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
586 {
587 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
588 AssertRCReturn(rc, rc);
589 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
590 }
591 return VINF_SUCCESS;
592}
593
594
595/**
596 * Reads the VM-exit interruption error code from the VMCS into the VMX
597 * transient structure.
598 *
599 * @returns VBox status code.
600 * @param pVCpu Pointer to the VMCPU.
601 * @param pVmxTransient Pointer to the VMX transient structure.
602 */
603DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
604{
605 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
606 {
607 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
608 AssertRCReturn(rc, rc);
609 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
610 }
611 return VINF_SUCCESS;
612}
613
614
615/**
616 * Reads the VM-exit instruction length field from the VMCS into the VMX
617 * transient structure.
618 *
619 * @returns VBox status code.
620 * @param pVCpu Pointer to the VMCPU.
621 * @param pVmxTransient Pointer to the VMX transient structure.
622 */
623DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
624{
625 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
626 {
627 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
628 AssertRCReturn(rc, rc);
629 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
630 }
631 return VINF_SUCCESS;
632}
633
634
635/**
636 * Reads the VM-exit instruction-information field from the VMCS into
637 * the VMX transient structure.
638 *
639 * @returns VBox status code.
640 * @param pVCpu The cross context per CPU structure.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
648 AssertRCReturn(rc, rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the exit qualification from the VMCS into the VMX transient structure.
657 *
658 * @returns VBox status code.
659 * @param pVCpu Pointer to the VMCPU.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
665 {
666 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the IDT-vectoring information field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 *
681 * @remarks No-long-jump zone!!!
682 */
683DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
684{
685 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
686 {
687 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
688 AssertRCReturn(rc, rc);
689 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
690 }
691 return VINF_SUCCESS;
692}
693
694
695/**
696 * Reads the IDT-vectoring error code from the VMCS into the VMX
697 * transient structure.
698 *
699 * @returns VBox status code.
700 * @param pVmxTransient Pointer to the VMX transient structure.
701 */
702DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
703{
704 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
705 {
706 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
707 AssertRCReturn(rc, rc);
708 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
709 }
710 return VINF_SUCCESS;
711}
712
713
714/**
715 * Enters VMX root mode operation on the current CPU.
716 *
717 * @returns VBox status code.
718 * @param pVM Pointer to the VM (optional, can be NULL, after
719 * a resume).
720 * @param HCPhysCpuPage Physical address of the VMXON region.
721 * @param pvCpuPage Pointer to the VMXON region.
722 */
723static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
724{
725 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
726 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
727 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
728
729 if (pVM)
730 {
731 /* Write the VMCS revision dword to the VMXON region. */
732 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
733 }
734
735 /* Enable the VMX bit in CR4 if necessary. */
736 RTCCUINTREG uCr4 = ASMGetCR4();
737 if (!(uCr4 & X86_CR4_VMXE))
738 ASMSetCR4(uCr4 | X86_CR4_VMXE);
739
740 /* Enter VMX root mode. */
741 int rc = VMXEnable(HCPhysCpuPage);
742 if (RT_FAILURE(rc))
743 ASMSetCR4(uCr4);
744
745 return rc;
746}
747
748
749/**
750 * Exits VMX root mode operation on the current CPU.
751 *
752 * @returns VBox status code.
753 */
754static int hmR0VmxLeaveRootMode(void)
755{
756 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
757
758 /* If we're for some reason not in VMX root mode, then don't leave it. */
759 RTCCUINTREG uHostCR4 = ASMGetCR4();
760 if (uHostCR4 & X86_CR4_VMXE)
761 {
762 /* Exit VMX root mode and clear the VMX bit in CR4. */
763 VMXDisable();
764 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
765 return VINF_SUCCESS;
766 }
767
768 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
769}
770
771
772/**
773 * Allocates and maps one physically contiguous page. The allocated page is
774 * zero'd out. (Used by various VT-x structures).
775 *
776 * @returns IPRT status code.
777 * @param pMemObj Pointer to the ring-0 memory object.
778 * @param ppVirt Where to store the virtual address of the
779 * allocation.
780 * @param pPhys Where to store the physical address of the
781 * allocation.
782 */
783DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
784{
785 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
786 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
787 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
788
789 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
790 if (RT_FAILURE(rc))
791 return rc;
792 *ppVirt = RTR0MemObjAddress(*pMemObj);
793 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
794 ASMMemZero32(*ppVirt, PAGE_SIZE);
795 return VINF_SUCCESS;
796}
797
798
799/**
800 * Frees and unmaps an allocated physical page.
801 *
802 * @param pMemObj Pointer to the ring-0 memory object.
803 * @param ppVirt Where to re-initialize the virtual address of
804 * allocation as 0.
805 * @param pHCPhys Where to re-initialize the physical address of the
806 * allocation as 0.
807 */
808DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
809{
810 AssertPtr(pMemObj);
811 AssertPtr(ppVirt);
812 AssertPtr(pHCPhys);
813 if (*pMemObj != NIL_RTR0MEMOBJ)
814 {
815 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
816 AssertRC(rc);
817 *pMemObj = NIL_RTR0MEMOBJ;
818 *ppVirt = 0;
819 *pHCPhys = 0;
820 }
821}
822
823
824/**
825 * Worker function to free VT-x related structures.
826 *
827 * @returns IPRT status code.
828 * @param pVM Pointer to the VM.
829 */
830static void hmR0VmxStructsFree(PVM pVM)
831{
832 for (VMCPUID i = 0; i < pVM->cCpus; i++)
833 {
834 PVMCPU pVCpu = &pVM->aCpus[i];
835 AssertPtr(pVCpu);
836
837#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
838 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
839 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
840#endif
841
842 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
843 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
844
845 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
846 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
847 }
848
849 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
850#ifdef VBOX_WITH_CRASHDUMP_MAGIC
851 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
852#endif
853}
854
855
856/**
857 * Worker function to allocate VT-x related VM structures.
858 *
859 * @returns IPRT status code.
860 * @param pVM Pointer to the VM.
861 */
862static int hmR0VmxStructsAlloc(PVM pVM)
863{
864 /*
865 * Initialize members up-front so we can cleanup properly on allocation failure.
866 */
867#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
868 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
869 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
870 pVM->hm.s.vmx.HCPhys##a_Name = 0;
871
872#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
873 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
874 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
875 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
876
877#ifdef VBOX_WITH_CRASHDUMP_MAGIC
878 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
879#endif
880 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
881
882 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
883 for (VMCPUID i = 0; i < pVM->cCpus; i++)
884 {
885 PVMCPU pVCpu = &pVM->aCpus[i];
886 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
887 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
888 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
889#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
890 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
891 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
892#endif
893 }
894#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
895#undef VMXLOCAL_INIT_VM_MEMOBJ
896
897 /*
898 * Allocate all the VT-x structures.
899 */
900 int rc = VINF_SUCCESS;
901#ifdef VBOX_WITH_CRASHDUMP_MAGIC
902 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
903 if (RT_FAILURE(rc))
904 goto cleanup;
905 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
906 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
907#endif
908
909 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
910 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
911 {
912 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
913 &pVM->hm.s.vmx.HCPhysApicAccess);
914 if (RT_FAILURE(rc))
915 goto cleanup;
916 }
917
918 /*
919 * Initialize per-VCPU VT-x structures.
920 */
921 for (VMCPUID i = 0; i < pVM->cCpus; i++)
922 {
923 PVMCPU pVCpu = &pVM->aCpus[i];
924 AssertPtr(pVCpu);
925
926 /* Allocate the VM control structure (VMCS). */
927 AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
928 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
929 if (RT_FAILURE(rc))
930 goto cleanup;
931
932 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
933 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
934 {
935 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
936 &pVCpu->hm.s.vmx.HCPhysVirtApic);
937 if (RT_FAILURE(rc))
938 goto cleanup;
939 }
940
941 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
942 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
943 {
944 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
945 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
946 if (RT_FAILURE(rc))
947 goto cleanup;
948 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
949 }
950
951#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
952 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
953 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
954 if (RT_FAILURE(rc))
955 goto cleanup;
956
957 /* Allocate the VM-exit MSR-load page for the host MSRs. */
958 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
959 if (RT_FAILURE(rc))
960 goto cleanup;
961#endif
962 }
963
964 return VINF_SUCCESS;
965
966cleanup:
967 hmR0VmxStructsFree(pVM);
968 return rc;
969}
970
971
972/**
973 * Does global VT-x initialization (called during module initialization).
974 *
975 * @returns VBox status code.
976 */
977VMMR0DECL(int) VMXR0GlobalInit(void)
978{
979#ifdef HMVMX_USE_FUNCTION_TABLE
980 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
981# ifdef VBOX_STRICT
982 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
983 Assert(g_apfnVMExitHandlers[i]);
984# endif
985#endif
986 return VINF_SUCCESS;
987}
988
989
990/**
991 * Does global VT-x termination (called during module termination).
992 */
993VMMR0DECL(void) VMXR0GlobalTerm()
994{
995 /* Nothing to do currently. */
996}
997
998
999/**
1000 * Sets up and activates VT-x on the current CPU.
1001 *
1002 * @returns VBox status code.
1003 * @param pCpu Pointer to the global CPU info struct.
1004 * @param pVM Pointer to the VM (can be NULL after a host resume
1005 * operation).
1006 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1007 * fEnabledByHost is true).
1008 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1009 * @a fEnabledByHost is true).
1010 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1011 * enable VT-x on the host.
1012 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1013 */
1014VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1015 void *pvMsrs)
1016{
1017 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1018 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1019 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1020
1021 /* Enable VT-x if it's not already enabled by the host. */
1022 if (!fEnabledByHost)
1023 {
1024 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1025 if (RT_FAILURE(rc))
1026 return rc;
1027 }
1028
1029 /*
1030 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1031 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1032 */
1033 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1034 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1035 {
1036 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1037 pCpu->fFlushAsidBeforeUse = false;
1038 }
1039 else
1040 pCpu->fFlushAsidBeforeUse = true;
1041
1042 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1043 ++pCpu->cTlbFlushes;
1044
1045 return VINF_SUCCESS;
1046}
1047
1048
1049/**
1050 * Deactivates VT-x on the current CPU.
1051 *
1052 * @returns VBox status code.
1053 * @param pCpu Pointer to the global CPU info struct.
1054 * @param pvCpuPage Pointer to the VMXON region.
1055 * @param HCPhysCpuPage Physical address of the VMXON region.
1056 *
1057 * @remarks This function should never be called when SUPR0EnableVTx() or
1058 * similar was used to enable VT-x on the host.
1059 */
1060VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1061{
1062 NOREF(pCpu);
1063 NOREF(pvCpuPage);
1064 NOREF(HCPhysCpuPage);
1065
1066 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1067 return hmR0VmxLeaveRootMode();
1068}
1069
1070
1071/**
1072 * Sets the permission bits for the specified MSR in the MSR bitmap.
1073 *
1074 * @param pVCpu Pointer to the VMCPU.
1075 * @param uMSR The MSR value.
1076 * @param enmRead Whether reading this MSR causes a VM-exit.
1077 * @param enmWrite Whether writing this MSR causes a VM-exit.
1078 */
1079static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1080{
1081 int32_t iBit;
1082 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1083
1084 /*
1085 * Layout:
1086 * 0x000 - 0x3ff - Low MSR read bits
1087 * 0x400 - 0x7ff - High MSR read bits
1088 * 0x800 - 0xbff - Low MSR write bits
1089 * 0xc00 - 0xfff - High MSR write bits
1090 */
1091 if (uMsr <= 0x00001FFF)
1092 iBit = uMsr;
1093 else if ( uMsr >= 0xC0000000
1094 && uMsr <= 0xC0001FFF)
1095 {
1096 iBit = (uMsr - 0xC0000000);
1097 pbMsrBitmap += 0x400;
1098 }
1099 else
1100 {
1101 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1102 return;
1103 }
1104
1105 Assert(iBit <= 0x1fff);
1106 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1107 ASMBitSet(pbMsrBitmap, iBit);
1108 else
1109 ASMBitClear(pbMsrBitmap, iBit);
1110
1111 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1112 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1113 else
1114 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1115}
1116
1117
1118/**
1119 * Flushes the TLB using EPT.
1120 *
1121 * @returns VBox status code.
1122 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1123 * enmFlush).
1124 * @param enmFlush Type of flush.
1125 *
1126 * @remarks Caller is responsible for making sure this function is called only
1127 * when NestedPaging is supported and providing @a enmFlush that is
1128 * supported by the CPU.
1129 */
1130static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1131{
1132 uint64_t au64Descriptor[2];
1133 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1134 au64Descriptor[0] = 0;
1135 else
1136 {
1137 Assert(pVCpu);
1138 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1139 }
1140 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1141
1142 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1143 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1144 rc));
1145 if ( RT_SUCCESS(rc)
1146 && pVCpu)
1147 {
1148 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1149 }
1150}
1151
1152
1153/**
1154 * Flushes the TLB using VPID.
1155 *
1156 * @returns VBox status code.
1157 * @param pVM Pointer to the VM.
1158 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1159 * enmFlush).
1160 * @param enmFlush Type of flush.
1161 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1162 * on @a enmFlush).
1163 */
1164static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1165{
1166 AssertPtr(pVM);
1167 Assert(pVM->hm.s.vmx.fVpid);
1168
1169 uint64_t au64Descriptor[2];
1170 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1171 {
1172 au64Descriptor[0] = 0;
1173 au64Descriptor[1] = 0;
1174 }
1175 else
1176 {
1177 AssertPtr(pVCpu);
1178 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1179 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1180 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1181 au64Descriptor[1] = GCPtr;
1182 }
1183
1184 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1185 AssertMsg(rc == VINF_SUCCESS,
1186 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1187 if ( RT_SUCCESS(rc)
1188 && pVCpu)
1189 {
1190 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1191 }
1192}
1193
1194
1195/**
1196 * Invalidates a guest page by guest virtual address. Only relevant for
1197 * EPT/VPID, otherwise there is nothing really to invalidate.
1198 *
1199 * @returns VBox status code.
1200 * @param pVM Pointer to the VM.
1201 * @param pVCpu Pointer to the VMCPU.
1202 * @param GCVirt Guest virtual address of the page to invalidate.
1203 */
1204VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1205{
1206 AssertPtr(pVM);
1207 AssertPtr(pVCpu);
1208 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1209
1210 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1211 if (!fFlushPending)
1212 {
1213 /*
1214 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1215 * See @bugref{6043} and @bugref{6177}.
1216 *
1217 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1218 * function maybe called in a loop with individual addresses.
1219 */
1220 if (pVM->hm.s.vmx.fVpid)
1221 {
1222 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1223 {
1224 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1225 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1226 }
1227 else
1228 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1229 }
1230 else if (pVM->hm.s.fNestedPaging)
1231 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1232 }
1233
1234 return VINF_SUCCESS;
1235}
1236
1237
1238/**
1239 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1240 * otherwise there is nothing really to invalidate.
1241 *
1242 * @returns VBox status code.
1243 * @param pVM Pointer to the VM.
1244 * @param pVCpu Pointer to the VMCPU.
1245 * @param GCPhys Guest physical address of the page to invalidate.
1246 */
1247VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1248{
1249 LogFlowFunc(("%RGp\n", GCPhys));
1250
1251 /*
1252 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1253 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1254 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1255 */
1256 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1257 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1258 return VINF_SUCCESS;
1259}
1260
1261
1262/**
1263 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1264 * case where neither EPT nor VPID is supported by the CPU.
1265 *
1266 * @param pVM Pointer to the VM.
1267 * @param pVCpu Pointer to the VMCPU.
1268 * @param pCpu Pointer to the global HM struct.
1269 *
1270 * @remarks Called with interrupts disabled.
1271 */
1272static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1273{
1274 AssertPtr(pVCpu);
1275 AssertPtr(pCpu);
1276 NOREF(pVM);
1277
1278 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1279 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1280
1281 pVCpu->hm.s.TlbShootdown.cPages = 0;
1282 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1283 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1284 pVCpu->hm.s.fForceTLBFlush = false;
1285 return;
1286}
1287
1288
1289/**
1290 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1291 *
1292 * @param pVM Pointer to the VM.
1293 * @param pVCpu Pointer to the VMCPU.
1294 * @param pCpu Pointer to the global HM CPU struct.
1295 * @remarks All references to "ASID" in this function pertains to "VPID" in
1296 * Intel's nomenclature. The reason is, to avoid confusion in compare
1297 * statements since the host-CPU copies are named "ASID".
1298 *
1299 * @remarks Called with interrupts disabled.
1300 */
1301static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1302{
1303#ifdef VBOX_WITH_STATISTICS
1304 bool fTlbFlushed = false;
1305# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1306# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1307 if (!fTlbFlushed) \
1308 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1309 } while (0)
1310#else
1311# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1312# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1313#endif
1314
1315 AssertPtr(pVM);
1316 AssertPtr(pCpu);
1317 AssertPtr(pVCpu);
1318 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1319 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1320 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1321
1322
1323 /*
1324 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1325 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1326 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1327 */
1328 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1329 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1330 {
1331 ++pCpu->uCurrentAsid;
1332 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1333 {
1334 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1335 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1336 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1337 }
1338
1339 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1340 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1341 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1342
1343 /*
1344 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1345 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1346 */
1347 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1348 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1349 HMVMX_SET_TAGGED_TLB_FLUSHED();
1350 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1351 }
1352
1353 /* Check for explicit TLB shootdowns. */
1354 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1355 {
1356 /*
1357 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1358 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1359 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1360 * but not guest-physical mappings.
1361 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1362 */
1363 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1364 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1365 HMVMX_SET_TAGGED_TLB_FLUSHED();
1366 }
1367
1368 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1369 * not be executed. See hmQueueInvlPage() where it is commented
1370 * out. Support individual entry flushing someday. */
1371 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1372 {
1373 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1374
1375 /*
1376 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1377 * as supported by the CPU.
1378 */
1379 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1380 {
1381 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1382 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1383 }
1384 else
1385 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1386
1387 HMVMX_SET_TAGGED_TLB_FLUSHED();
1388 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1389 }
1390
1391 pVCpu->hm.s.TlbShootdown.cPages = 0;
1392 pVCpu->hm.s.fForceTLBFlush = false;
1393
1394 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1395
1396 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1397 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1398 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1399 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1400 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1401 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1402 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1403 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1404
1405 /* Update VMCS with the VPID. */
1406 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1407 AssertRC(rc);
1408
1409#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1410}
1411
1412
1413/**
1414 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1415 *
1416 * @returns VBox status code.
1417 * @param pVM Pointer to the VM.
1418 * @param pVCpu Pointer to the VMCPU.
1419 * @param pCpu Pointer to the global HM CPU struct.
1420 *
1421 * @remarks Called with interrupts disabled.
1422 */
1423static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1424{
1425 AssertPtr(pVM);
1426 AssertPtr(pVCpu);
1427 AssertPtr(pCpu);
1428 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1429 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1430
1431 /*
1432 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1433 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1434 */
1435 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1436 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1437 {
1438 pVCpu->hm.s.fForceTLBFlush = true;
1439 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1440 }
1441
1442 /* Check for explicit TLB shootdown flushes. */
1443 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1444 {
1445 pVCpu->hm.s.fForceTLBFlush = true;
1446 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1447 }
1448
1449 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1450 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1451
1452 if (pVCpu->hm.s.fForceTLBFlush)
1453 {
1454 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1455 pVCpu->hm.s.fForceTLBFlush = false;
1456 }
1457 else
1458 {
1459 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1460 * not be executed. See hmQueueInvlPage() where it is commented
1461 * out. Support individual entry flushing someday. */
1462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1463 {
1464 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1465 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1466 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1467 }
1468 else
1469 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1470 }
1471
1472 pVCpu->hm.s.TlbShootdown.cPages = 0;
1473 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1474}
1475
1476
1477/**
1478 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1479 *
1480 * @returns VBox status code.
1481 * @param pVM Pointer to the VM.
1482 * @param pVCpu Pointer to the VMCPU.
1483 * @param pCpu Pointer to the global HM CPU struct.
1484 *
1485 * @remarks Called with interrupts disabled.
1486 */
1487static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1488{
1489 AssertPtr(pVM);
1490 AssertPtr(pVCpu);
1491 AssertPtr(pCpu);
1492 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1493 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1494
1495 /*
1496 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1497 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1498 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1499 */
1500 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1501 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1502 {
1503 pVCpu->hm.s.fForceTLBFlush = true;
1504 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1505 }
1506
1507 /* Check for explicit TLB shootdown flushes. */
1508 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1509 {
1510 /*
1511 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1512 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1513 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1514 */
1515 pVCpu->hm.s.fForceTLBFlush = true;
1516 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1517 }
1518
1519 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1520 if (pVCpu->hm.s.fForceTLBFlush)
1521 {
1522 ++pCpu->uCurrentAsid;
1523 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1524 {
1525 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1526 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1527 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1528 }
1529
1530 pVCpu->hm.s.fForceTLBFlush = false;
1531 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1532 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1533 if (pCpu->fFlushAsidBeforeUse)
1534 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1535 }
1536 else
1537 {
1538 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1539 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1540 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1541 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1542
1543 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1544 * not be executed. See hmQueueInvlPage() where it is commented
1545 * out. Support individual entry flushing someday. */
1546 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1547 {
1548 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1549 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1550 {
1551 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1552 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1553 }
1554 else
1555 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1556 }
1557 else
1558 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1559 }
1560
1561 pVCpu->hm.s.TlbShootdown.cPages = 0;
1562 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1563
1564 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1565 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1566 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1567 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1568 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1569 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1570
1571 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1572 AssertRC(rc);
1573}
1574
1575
1576/**
1577 * Flushes the guest TLB entry based on CPU capabilities.
1578 *
1579 * @param pVCpu Pointer to the VMCPU.
1580 * @param pCpu Pointer to the global HM CPU struct.
1581 */
1582DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1583{
1584 PVM pVM = pVCpu->CTX_SUFF(pVM);
1585 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1586 {
1587 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1588 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1589 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1590 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1591 default:
1592 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1593 break;
1594 }
1595}
1596
1597
1598/**
1599 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1600 * TLB entries from the host TLB before VM-entry.
1601 *
1602 * @returns VBox status code.
1603 * @param pVM Pointer to the VM.
1604 */
1605static int hmR0VmxSetupTaggedTlb(PVM pVM)
1606{
1607 /*
1608 * Determine optimal flush type for Nested Paging.
1609 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1610 * guest execution (see hmR3InitFinalizeR0()).
1611 */
1612 if (pVM->hm.s.fNestedPaging)
1613 {
1614 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1615 {
1616 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1617 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1618 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1619 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1620 else
1621 {
1622 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1623 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1624 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1625 }
1626
1627 /* Make sure the write-back cacheable memory type for EPT is supported. */
1628 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1629 {
1630 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1631 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1632 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1633 }
1634 }
1635 else
1636 {
1637 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1638 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1639 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1640 }
1641 }
1642
1643 /*
1644 * Determine optimal flush type for VPID.
1645 */
1646 if (pVM->hm.s.vmx.fVpid)
1647 {
1648 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1649 {
1650 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1651 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1652 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1653 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1654 else
1655 {
1656 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1657 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1658 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1659 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1660 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1661 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1662 pVM->hm.s.vmx.fVpid = false;
1663 }
1664 }
1665 else
1666 {
1667 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1668 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1669 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1670 pVM->hm.s.vmx.fVpid = false;
1671 }
1672 }
1673
1674 /*
1675 * Setup the handler for flushing tagged-TLBs.
1676 */
1677 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1678 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1679 else if (pVM->hm.s.fNestedPaging)
1680 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1681 else if (pVM->hm.s.vmx.fVpid)
1682 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1683 else
1684 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1685 return VINF_SUCCESS;
1686}
1687
1688
1689/**
1690 * Sets up pin-based VM-execution controls in the VMCS.
1691 *
1692 * @returns VBox status code.
1693 * @param pVM Pointer to the VM.
1694 * @param pVCpu Pointer to the VMCPU.
1695 */
1696static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1697{
1698 AssertPtr(pVM);
1699 AssertPtr(pVCpu);
1700
1701 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1702 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1703
1704 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1705 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1706 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1707
1708 /* Enable the VMX preemption timer. */
1709 if (pVM->hm.s.vmx.fUsePreemptTimer)
1710 {
1711 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1712 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1713 }
1714
1715 if ((val & zap) != val)
1716 {
1717 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1718 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1719 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1720 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1721 }
1722
1723 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1724 AssertRCReturn(rc, rc);
1725
1726 /* Update VCPU with the currently set pin-based VM-execution controls. */
1727 pVCpu->hm.s.vmx.u32PinCtls = val;
1728 return rc;
1729}
1730
1731
1732/**
1733 * Sets up processor-based VM-execution controls in the VMCS.
1734 *
1735 * @returns VBox status code.
1736 * @param pVM Pointer to the VM.
1737 * @param pVMCPU Pointer to the VMCPU.
1738 */
1739static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1740{
1741 AssertPtr(pVM);
1742 AssertPtr(pVCpu);
1743
1744 int rc = VERR_INTERNAL_ERROR_5;
1745 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1746 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1747
1748 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1749 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1750 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1751 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1752 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1753 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1754 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1755
1756 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1757 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1758 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1759 {
1760 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1761 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1762 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1763 }
1764
1765 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1766 if (!pVM->hm.s.fNestedPaging)
1767 {
1768 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1769 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1770 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1771 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1772 }
1773
1774 /* Use TPR shadowing if supported by the CPU. */
1775 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1776 {
1777 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1778 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1779 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1780 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1781 AssertRCReturn(rc, rc);
1782
1783 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1784 /* CR8 writes causes a VM-exit based on TPR threshold. */
1785 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1786 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1787 }
1788 else
1789 {
1790 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1791 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1792 }
1793
1794 /* Use MSR-bitmaps if supported by the CPU. */
1795 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1796 {
1797 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1798
1799 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1800 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1801 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1802 AssertRCReturn(rc, rc);
1803
1804 /*
1805 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1806 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1807 */
1808 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1809 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1810 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1811 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1812 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1813 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1814 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1815 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1816 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1817 }
1818
1819 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1820 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1821 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1822
1823 if ((val & zap) != val)
1824 {
1825 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1826 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1827 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1828 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1829 }
1830
1831 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1832 AssertRCReturn(rc, rc);
1833
1834 /* Update VCPU with the currently set processor-based VM-execution controls. */
1835 pVCpu->hm.s.vmx.u32ProcCtls = val;
1836
1837 /*
1838 * Secondary processor-based VM-execution controls.
1839 */
1840 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1841 {
1842 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1843 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1844
1845 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1846 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1847
1848 if (pVM->hm.s.fNestedPaging)
1849 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1850 else
1851 {
1852 /*
1853 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1854 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1855 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1856 */
1857 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1858 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1859 }
1860
1861 if (pVM->hm.s.vmx.fVpid)
1862 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1863
1864 if (pVM->hm.s.vmx.fUnrestrictedGuest)
1865 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1866
1867 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1868 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1869 * done dynamically. */
1870 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1871 {
1872 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1873 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1874 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1875 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1876 AssertRCReturn(rc, rc);
1877 }
1878
1879 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1880 {
1881 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1882 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1883 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1884 }
1885
1886 if ((val & zap) != val)
1887 {
1888 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1889 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1890 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1891 }
1892
1893 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1894 AssertRCReturn(rc, rc);
1895
1896 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1897 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1898 }
1899 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1900 {
1901 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1902 "available\n"));
1903 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1904 }
1905
1906 return VINF_SUCCESS;
1907}
1908
1909
1910/**
1911 * Sets up miscellaneous (everything other than Pin & Processor-based
1912 * VM-execution) control fields in the VMCS.
1913 *
1914 * @returns VBox status code.
1915 * @param pVM Pointer to the VM.
1916 * @param pVCpu Pointer to the VMCPU.
1917 */
1918static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1919{
1920 AssertPtr(pVM);
1921 AssertPtr(pVCpu);
1922
1923 int rc = VERR_GENERAL_FAILURE;
1924
1925 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1926#if 0
1927 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1928 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1929 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1930
1931 /*
1932 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1933 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1934 * We thus use the exception bitmap to control it rather than use both.
1935 */
1936 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1937 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1938
1939 /** @todo Explore possibility of using IO-bitmaps. */
1940 /* All IO & IOIO instructions cause VM-exits. */
1941 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1942 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1943
1944 /* Initialize the MSR-bitmap area. */
1945 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1946 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1947 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1948#endif
1949
1950#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1951 /* Setup MSR autoloading/storing. */
1952 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1953 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1954 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1955 AssertRCReturn(rc, rc);
1956 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1957 AssertRCReturn(rc, rc);
1958
1959 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1960 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1961 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1962 AssertRCReturn(rc, rc);
1963#endif
1964
1965 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1966 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1967 AssertRCReturn(rc, rc);
1968
1969 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1970#if 0
1971 /* Setup debug controls */
1972 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1973 AssertRCReturn(rc, rc);
1974 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1975 AssertRCReturn(rc, rc);
1976#endif
1977
1978 return rc;
1979}
1980
1981
1982/**
1983 * Sets up the initial exception bitmap in the VMCS based on static conditions
1984 * (i.e. conditions that cannot ever change after starting the VM).
1985 *
1986 * @returns VBox status code.
1987 * @param pVM Pointer to the VM.
1988 * @param pVCpu Pointer to the VMCPU.
1989 */
1990static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1991{
1992 AssertPtr(pVM);
1993 AssertPtr(pVCpu);
1994
1995 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1996
1997 uint32_t u32XcptBitmap = 0;
1998
1999 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2000 if (!pVM->hm.s.fNestedPaging)
2001 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2002
2003 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2004 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2005 AssertRCReturn(rc, rc);
2006 return rc;
2007}
2008
2009
2010/**
2011 * Sets up the initial guest-state mask. The guest-state mask is consulted
2012 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2013 * for the nested virtualization case (as it would cause a VM-exit).
2014 *
2015 * @param pVCpu Pointer to the VMCPU.
2016 */
2017static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2018{
2019 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2020 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2021 return VINF_SUCCESS;
2022}
2023
2024
2025/**
2026 * Does per-VM VT-x initialization.
2027 *
2028 * @returns VBox status code.
2029 * @param pVM Pointer to the VM.
2030 */
2031VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2032{
2033 LogFlowFunc(("pVM=%p\n", pVM));
2034
2035 int rc = hmR0VmxStructsAlloc(pVM);
2036 if (RT_FAILURE(rc))
2037 {
2038 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2039 return rc;
2040 }
2041
2042 return VINF_SUCCESS;
2043}
2044
2045
2046/**
2047 * Does per-VM VT-x termination.
2048 *
2049 * @returns VBox status code.
2050 * @param pVM Pointer to the VM.
2051 */
2052VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2053{
2054 LogFlowFunc(("pVM=%p\n", pVM));
2055
2056#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2057 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2058 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2059#endif
2060 hmR0VmxStructsFree(pVM);
2061 return VINF_SUCCESS;
2062}
2063
2064
2065/**
2066 * Sets up the VM for execution under VT-x.
2067 * This function is only called once per-VM during initialization.
2068 *
2069 * @returns VBox status code.
2070 * @param pVM Pointer to the VM.
2071 */
2072VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2073{
2074 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2075 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2076
2077 LogFlowFunc(("pVM=%p\n", pVM));
2078
2079 /*
2080 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2081 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2082 */
2083 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2084 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2085 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2086 || !pVM->hm.s.vmx.pRealModeTSS))
2087 {
2088 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2089 return VERR_INTERNAL_ERROR;
2090 }
2091
2092#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2093 /*
2094 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2095 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2096 */
2097 if ( pVM->hm.s.fAllow64BitGuests
2098 && !HMVMX_IS_64BIT_HOST_MODE())
2099 {
2100 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2101 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2102 }
2103#endif
2104
2105 /* Initialize these always, see hmR3InitFinalizeR0().*/
2106 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2107 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2108
2109 /* Setup the tagged-TLB flush handlers. */
2110 int rc = hmR0VmxSetupTaggedTlb(pVM);
2111 if (RT_FAILURE(rc))
2112 {
2113 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2114 return rc;
2115 }
2116
2117 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2118 {
2119 PVMCPU pVCpu = &pVM->aCpus[i];
2120 AssertPtr(pVCpu);
2121 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2122
2123 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2124 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2125
2126 /* Set revision dword at the beginning of the VMCS structure. */
2127 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2128
2129 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2130 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2131 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2132 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2133
2134 /* Load this VMCS as the current VMCS. */
2135 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2136 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2137 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2138
2139 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2140 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2141 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2142
2143 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2144 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2145 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2146
2147 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2148 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2149 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2150
2151 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2152 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2153 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2154
2155 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2156 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2157 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2158
2159#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2160 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2161 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2162 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2163#endif
2164
2165 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2166 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2167 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2168 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2169
2170 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2171
2172 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2173 }
2174
2175 return VINF_SUCCESS;
2176}
2177
2178
2179/**
2180 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2181 * the VMCS.
2182 *
2183 * @returns VBox status code.
2184 * @param pVM Pointer to the VM.
2185 * @param pVCpu Pointer to the VMCPU.
2186 */
2187DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2188{
2189 RTCCUINTREG uReg = ASMGetCR0();
2190 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2191 AssertRCReturn(rc, rc);
2192
2193#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2194 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2195 if (HMVMX_IS_64BIT_HOST_MODE())
2196 {
2197 uint64_t uRegCR3 = HMR0Get64bitCR3();
2198 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2199 }
2200 else
2201#endif
2202 {
2203 uReg = ASMGetCR3();
2204 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2205 }
2206 AssertRCReturn(rc, rc);
2207
2208 uReg = ASMGetCR4();
2209 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2210 AssertRCReturn(rc, rc);
2211 return rc;
2212}
2213
2214
2215/**
2216 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2217 * the host-state area in the VMCS.
2218 *
2219 * @returns VBox status code.
2220 * @param pVM Pointer to the VM.
2221 * @param pVCpu Pointer to the VMCPU.
2222 */
2223DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2224{
2225 int rc = VERR_INTERNAL_ERROR_5;
2226 RTSEL uSelDS = 0;
2227 RTSEL uSelES = 0;
2228 RTSEL uSelFS = 0;
2229 RTSEL uSelGS = 0;
2230 RTSEL uSelTR = 0;
2231
2232 /*
2233 * Host DS, ES, FS and GS segment registers.
2234 */
2235#if HC_ARCH_BITS == 64
2236 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2237 uSelDS = ASMGetDS();
2238 uSelES = ASMGetES();
2239 uSelFS = ASMGetFS();
2240 uSelGS = ASMGetGS();
2241#endif
2242
2243 /*
2244 * Host CS and SS segment registers.
2245 */
2246 RTSEL uSelCS;
2247 RTSEL uSelSS;
2248#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2249 if (HMVMX_IS_64BIT_HOST_MODE())
2250 {
2251 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2252 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2253 }
2254 else
2255 {
2256 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2257 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2258 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2259 }
2260#else
2261 uSelCS = ASMGetCS();
2262 uSelSS = ASMGetSS();
2263#endif
2264
2265 /*
2266 * Host TR segment register.
2267 */
2268 uSelTR = ASMGetTR();
2269
2270#if HC_ARCH_BITS == 64
2271 /*
2272 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2273 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2274 */
2275 if (uSelDS & (X86_SEL_RPL | X86_SEL_LDT))
2276 {
2277 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_DS;
2278 pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2279 uSelDS = 0;
2280 }
2281 if (uSelES & (X86_SEL_RPL | X86_SEL_LDT))
2282 {
2283 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_ES;
2284 pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2285 uSelES = 0;
2286 }
2287 if (uSelFS & (X86_SEL_RPL | X86_SEL_LDT))
2288 {
2289 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_FS;
2290 pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2291 uSelFS = 0;
2292 }
2293 if (uSelGS & (X86_SEL_RPL | X86_SEL_LDT))
2294 {
2295 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_GS;
2296 pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2297 uSelGS = 0;
2298 }
2299#endif
2300
2301 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2302 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2303 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2304 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2305 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2306 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2307 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2308 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2309 Assert(uSelCS);
2310 Assert(uSelTR);
2311
2312 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2313#if 0
2314 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2315 Assert(uSelSS != 0);
2316#endif
2317
2318 /* Write these host selector fields into the host-state area in the VMCS. */
2319 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2320 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2321#if HC_ARCH_BITS == 64
2322 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2323 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2324 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2325 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2326#endif
2327 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2328
2329 /*
2330 * Host GDTR and IDTR.
2331 */
2332 RTGDTR Gdtr;
2333 RT_ZERO(Gdtr);
2334#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2335 if (HMVMX_IS_64BIT_HOST_MODE())
2336 {
2337 X86XDTR64 Gdtr64;
2338 X86XDTR64 Idtr64;
2339 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2340 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2341 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2342
2343 Gdtr.cbGdt = Gdtr64.cb;
2344 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2345 }
2346 else
2347#endif
2348 {
2349 RTIDTR Idtr;
2350 ASMGetGDTR(&Gdtr);
2351 ASMGetIDTR(&Idtr);
2352 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2353 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2354
2355#if HC_ARCH_BITS == 64
2356 /*
2357 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2358 * maximum limit (0xffff) on every VM-exit.
2359 */
2360 if (Gdtr.cbGdt != 0xffff)
2361 {
2362 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2363 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2364 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2365 }
2366
2367 /*
2368 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2369 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2370 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2371 */
2372 if (Idtr.cbIdt < 0x0fff)
2373 {
2374 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2375 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2376 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2377 }
2378#endif
2379 }
2380
2381 /*
2382 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2383 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2384 */
2385 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2386 {
2387 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2388 return VERR_VMX_INVALID_HOST_STATE;
2389 }
2390
2391 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2392#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2393 if (HMVMX_IS_64BIT_HOST_MODE())
2394 {
2395 /* We need the 64-bit TR base for hybrid darwin. */
2396 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2397 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2398 }
2399 else
2400#endif
2401 {
2402 uintptr_t uTRBase;
2403#if HC_ARCH_BITS == 64
2404 uTRBase = X86DESC64_BASE(pDesc);
2405
2406 /*
2407 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2408 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2409 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2410 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2411 *
2412 * [1] See Intel spec. 3.5 "System Descriptor Types".
2413 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2414 */
2415 Assert(pDesc->System.u4Type == 11);
2416 if ( pDesc->System.u16LimitLow != 0x67
2417 || pDesc->System.u4LimitHigh)
2418 {
2419 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2420 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2421
2422 /* Store the GDTR here as we need it while restoring TR. */
2423 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2424 }
2425#else
2426 uTRBase = X86DESC_BASE(pDesc);
2427#endif
2428 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2429 }
2430 AssertRCReturn(rc, rc);
2431
2432 /*
2433 * Host FS base and GS base.
2434 */
2435#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2436 if (HMVMX_IS_64BIT_HOST_MODE())
2437 {
2438 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2439 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2440 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2441 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2442
2443# if HC_ARCH_BITS == 64
2444 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2445 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2446 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2447 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2448 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2449# endif
2450 }
2451#endif
2452 return rc;
2453}
2454
2455
2456/**
2457 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2458 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2459 * the host after every successful VM exit.
2460 *
2461 * @returns VBox status code.
2462 * @param pVM Pointer to the VM.
2463 * @param pVCpu Pointer to the VMCPU.
2464 */
2465DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2466{
2467 AssertPtr(pVCpu);
2468 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2469
2470 int rc = VINF_SUCCESS;
2471#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2472 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2473 uint32_t cHostMsrs = 0;
2474 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2475
2476 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2477 {
2478 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2479
2480# if HC_ARCH_BITS == 64
2481 /* Paranoia. 64-bit code requires these bits to be set always. */
2482 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2483
2484 /*
2485 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2486 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2487 * some reason (e.g. allow transparent reads) we would activate the code below.
2488 */
2489# if 0
2490 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2491 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2492 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2493 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2494 if (CPUMIsGuestInLongMode(pVCpu))
2495 {
2496 uint64_t u64GuestEfer;
2497 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2498 AssertRC(rc);
2499
2500 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2501 {
2502 pHostMsr->u32Msr = MSR_K6_EFER;
2503 pHostMsr->u32Reserved = 0;
2504 pHostMsr->u64Value = u64HostEfer;
2505 pHostMsr++; cHostMsrs++;
2506 }
2507 }
2508# endif
2509# else /* HC_ARCH_BITS != 64 */
2510 pHostMsr->u32Msr = MSR_K6_EFER;
2511 pHostMsr->u32Reserved = 0;
2512# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2513 if (CPUMIsGuestInLongMode(pVCpu))
2514 {
2515 /* Must match the EFER value in our 64 bits switcher. */
2516 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2517 }
2518 else
2519# endif
2520 pHostMsr->u64Value = u64HostEfer;
2521 pHostMsr++; cHostMsrs++;
2522# endif /* HC_ARCH_BITS == 64 */
2523 }
2524
2525# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2526 if (HMVMX_IS_64BIT_HOST_MODE())
2527 {
2528 pHostMsr->u32Msr = MSR_K6_STAR;
2529 pHostMsr->u32Reserved = 0;
2530 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2531 pHostMsr++; cHostMsrs++;
2532 pHostMsr->u32Msr = MSR_K8_LSTAR;
2533 pHostMsr->u32Reserved = 0;
2534 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2535 pHostMsr++; cHostMsrs++;
2536 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2537 pHostMsr->u32Reserved = 0;
2538 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2539 pHostMsr++; cHostMsrs++;
2540 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2541 pHostMsr->u32Reserved = 0;
2542 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2543 pHostMsr++; cHostMsrs++;
2544 }
2545# endif
2546
2547 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2548 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2549 {
2550 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2551 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2552 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2553 }
2554
2555 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2556#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2557
2558 /*
2559 * Host Sysenter MSRs.
2560 */
2561 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2562 AssertRCReturn(rc, rc);
2563#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2564 if (HMVMX_IS_64BIT_HOST_MODE())
2565 {
2566 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2567 AssertRCReturn(rc, rc);
2568 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2569 }
2570 else
2571 {
2572 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2573 AssertRCReturn(rc, rc);
2574 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2575 }
2576#elif HC_ARCH_BITS == 32
2577 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2578 AssertRCReturn(rc, rc);
2579 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2580#else
2581 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2582 AssertRCReturn(rc, rc);
2583 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2584#endif
2585 AssertRCReturn(rc, rc);
2586
2587 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2588 * hmR0VmxSetupExitCtls() !! */
2589 return rc;
2590}
2591
2592
2593/**
2594 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2595 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2596 * controls".
2597 *
2598 * @returns VBox status code.
2599 * @param pVCpu Pointer to the VMCPU.
2600 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2601 * out-of-sync. Make sure to update the required fields
2602 * before using them.
2603 *
2604 * @remarks No-long-jump zone!!!
2605 */
2606DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2607{
2608 int rc = VINF_SUCCESS;
2609 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2610 {
2611 PVM pVM = pVCpu->CTX_SUFF(pVM);
2612 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2613 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2614
2615 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2616 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2617
2618 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2619 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2620 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2621 else
2622 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2623
2624 /*
2625 * The following should not be set (since we're not in SMM mode):
2626 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2627 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2628 */
2629
2630 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2631 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2632 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2633
2634 if ((val & zap) != val)
2635 {
2636 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2637 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2638 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2639 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2640 }
2641
2642 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2643 AssertRCReturn(rc, rc);
2644
2645 /* Update VCPU with the currently set VM-exit controls. */
2646 pVCpu->hm.s.vmx.u32EntryCtls = val;
2647 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2648 }
2649 return rc;
2650}
2651
2652
2653/**
2654 * Sets up the VM-exit controls in the VMCS.
2655 *
2656 * @returns VBox status code.
2657 * @param pVM Pointer to the VM.
2658 * @param pVCpu Pointer to the VMCPU.
2659 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2660 * out-of-sync. Make sure to update the required fields
2661 * before using them.
2662 *
2663 * @remarks requires EFER.
2664 */
2665DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2666{
2667 int rc = VINF_SUCCESS;
2668 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2669 {
2670 PVM pVM = pVCpu->CTX_SUFF(pVM);
2671 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2672 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2673
2674 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2675 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2676
2677 /*
2678 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2679 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2680 */
2681#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2682 if (HMVMX_IS_64BIT_HOST_MODE())
2683 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2684 else
2685 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2686#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2687 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2688 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2689 else
2690 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2691#endif
2692
2693 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2694 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2695
2696 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2697 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2698 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2699 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2700 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2701
2702 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2703 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2704
2705 if ((val & zap) != val)
2706 {
2707 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2708 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2709 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2710 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2711 }
2712
2713 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2714 AssertRCReturn(rc, rc);
2715
2716 /* Update VCPU with the currently set VM-exit controls. */
2717 pVCpu->hm.s.vmx.u32ExitCtls = val;
2718 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2719 }
2720 return rc;
2721}
2722
2723
2724/**
2725 * Loads the guest APIC and related state.
2726 *
2727 * @returns VBox status code.
2728 * @param pVM Pointer to the VM.
2729 * @param pVCpu Pointer to the VMCPU.
2730 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2731 * out-of-sync. Make sure to update the required fields
2732 * before using them.
2733 */
2734DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2735{
2736 int rc = VINF_SUCCESS;
2737 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2738 {
2739 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2740 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2741 {
2742 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2743
2744 bool fPendingIntr = false;
2745 uint8_t u8Tpr = 0;
2746 uint8_t u8PendingIntr = 0;
2747 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2748 AssertRCReturn(rc, rc);
2749
2750 /*
2751 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2752 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2753 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2754 * the interrupt when we VM-exit for other reasons.
2755 */
2756 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2757 uint32_t u32TprThreshold = 0;
2758 if (fPendingIntr)
2759 {
2760 /* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2761 const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2762 const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2763 if (u8PendingPriority <= u8TprPriority)
2764 u32TprThreshold = u8PendingPriority;
2765 else
2766 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2767 }
2768 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2769
2770 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2771 AssertRCReturn(rc, rc);
2772 }
2773
2774 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2775 }
2776 return rc;
2777}
2778
2779
2780/**
2781 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2782 *
2783 * @returns Guest's interruptibility-state.
2784 * @param pVCpu Pointer to the VMCPU.
2785 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2786 * out-of-sync. Make sure to update the required fields
2787 * before using them.
2788 *
2789 * @remarks No-long-jump zone!!!
2790 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2791 */
2792DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2793{
2794 /*
2795 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2796 * inhibit interrupts or clear any existing interrupt-inhibition.
2797 */
2798 uint32_t uIntrState = 0;
2799 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2800 {
2801 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2802 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
2803 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2804 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2805 {
2806 /*
2807 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2808 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2809 */
2810 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2811 }
2812 else if (pMixedCtx->eflags.Bits.u1IF)
2813 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2814 else
2815 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2816 }
2817 return uIntrState;
2818}
2819
2820
2821/**
2822 * Loads the guest's interruptibility-state into the guest-state area in the
2823 * VMCS.
2824 *
2825 * @returns VBox status code.
2826 * @param pVCpu Pointer to the VMCPU.
2827 * @param uIntrState The interruptibility-state to set.
2828 */
2829static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2830{
2831 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2832 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2833 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2834 AssertRCReturn(rc, rc);
2835 return rc;
2836}
2837
2838
2839/**
2840 * Loads the guest's RIP into the guest-state area in the VMCS.
2841 *
2842 * @returns VBox status code.
2843 * @param pVCpu Pointer to the VMCPU.
2844 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2845 * out-of-sync. Make sure to update the required fields
2846 * before using them.
2847 *
2848 * @remarks No-long-jump zone!!!
2849 */
2850static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2851{
2852 int rc = VINF_SUCCESS;
2853 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2854 {
2855 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2856 AssertRCReturn(rc, rc);
2857 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2858 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#x\n", pMixedCtx->rip, pVCpu->hm.s.fContextUseFlags));
2859 }
2860 return rc;
2861}
2862
2863
2864/**
2865 * Loads the guest's RSP into the guest-state area in the VMCS.
2866 *
2867 * @returns VBox status code.
2868 * @param pVCpu Pointer to the VMCPU.
2869 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2870 * out-of-sync. Make sure to update the required fields
2871 * before using them.
2872 *
2873 * @remarks No-long-jump zone!!!
2874 */
2875static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2876{
2877 int rc = VINF_SUCCESS;
2878 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2879 {
2880 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2881 AssertRCReturn(rc, rc);
2882 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2883 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2884 }
2885 return rc;
2886}
2887
2888
2889/**
2890 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
2891 *
2892 * @returns VBox status code.
2893 * @param pVCpu Pointer to the VMCPU.
2894 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2895 * out-of-sync. Make sure to update the required fields
2896 * before using them.
2897 *
2898 * @remarks No-long-jump zone!!!
2899 */
2900static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2901{
2902 int rc = VINF_SUCCESS;
2903 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2904 {
2905 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2906 Let us assert it as such and use 32-bit VMWRITE. */
2907 Assert(!(pMixedCtx->rflags.u64 >> 32));
2908 X86EFLAGS Eflags = pMixedCtx->eflags;
2909 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2910 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2911
2912 /*
2913 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2914 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2915 */
2916 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2917 {
2918 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2919 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2920 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2921 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2922 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2923 }
2924
2925 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2926 AssertRCReturn(rc, rc);
2927
2928 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2929 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2930 }
2931 return rc;
2932}
2933
2934
2935/**
2936 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2937 *
2938 * @returns VBox status code.
2939 * @param pVCpu Pointer to the VMCPU.
2940 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2941 * out-of-sync. Make sure to update the required fields
2942 * before using them.
2943 *
2944 * @remarks No-long-jump zone!!!
2945 */
2946DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2947{
2948 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2949 AssertRCReturn(rc, rc);
2950 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2951 AssertRCReturn(rc, rc);
2952 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2953 AssertRCReturn(rc, rc);
2954 return rc;
2955}
2956
2957
2958/**
2959 * Loads the guest CR0 control register into the guest-state area in the VMCS.
2960 * CR0 is partially shared with the host and we have to consider the FPU bits.
2961 *
2962 * @returns VBox status code.
2963 * @param pVM Pointer to the VM.
2964 * @param pVCpu Pointer to the VMCPU.
2965 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2966 * out-of-sync. Make sure to update the required fields
2967 * before using them.
2968 *
2969 * @remarks No-long-jump zone!!!
2970 */
2971static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2972{
2973 /*
2974 * Guest CR0.
2975 * Guest FPU.
2976 */
2977 int rc = VINF_SUCCESS;
2978 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2979 {
2980 Assert(!(pMixedCtx->cr0 >> 32));
2981 uint32_t u32GuestCR0 = pMixedCtx->cr0;
2982 PVM pVM = pVCpu->CTX_SUFF(pVM);
2983
2984 /* The guest's view (read access) of its CR0 is unblemished. */
2985 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2986 AssertRCReturn(rc, rc);
2987 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2988
2989 /* Setup VT-x's view of the guest CR0. */
2990 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2991 if (pVM->hm.s.fNestedPaging)
2992 {
2993 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
2994 {
2995 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2996 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2997 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2998 }
2999 else
3000 {
3001 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3002 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3003 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3004 }
3005
3006 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3007 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3008 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3009
3010 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3011 AssertRCReturn(rc, rc);
3012 }
3013 else
3014 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3015
3016 /*
3017 * Guest FPU bits.
3018 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3019 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3020 */
3021 u32GuestCR0 |= X86_CR0_NE;
3022 bool fInterceptNM = false;
3023 if (CPUMIsGuestFPUStateActive(pVCpu))
3024 {
3025 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3026 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3027 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3028 }
3029 else
3030 {
3031 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3032 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3033 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3034 }
3035
3036 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3037 bool fInterceptMF = false;
3038 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3039 fInterceptMF = true;
3040
3041 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3042 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3043 {
3044 Assert(PDMVmmDevHeapIsEnabled(pVM));
3045 Assert(pVM->hm.s.vmx.pRealModeTSS);
3046 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3047 fInterceptNM = true;
3048 fInterceptMF = true;
3049 }
3050 else
3051 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3052
3053 if (fInterceptNM)
3054 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3055 else
3056 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3057
3058 if (fInterceptMF)
3059 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3060 else
3061 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3062
3063 /* Additional intercepts for debugging, define these yourself explicitly. */
3064#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3065 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3066 | RT_BIT(X86_XCPT_BP)
3067 | RT_BIT(X86_XCPT_DB)
3068 | RT_BIT(X86_XCPT_DE)
3069 | RT_BIT(X86_XCPT_NM)
3070 | RT_BIT(X86_XCPT_UD)
3071 | RT_BIT(X86_XCPT_NP)
3072 | RT_BIT(X86_XCPT_SS)
3073 | RT_BIT(X86_XCPT_GP)
3074 | RT_BIT(X86_XCPT_PF)
3075 | RT_BIT(X86_XCPT_MF)
3076 ;
3077#elif defined(HMVMX_ALWAYS_TRAP_PF)
3078 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3079#endif
3080
3081 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3082
3083 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3084 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3085 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3086 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3087 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3088 else
3089 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3090
3091 u32GuestCR0 |= uSetCR0;
3092 u32GuestCR0 &= uZapCR0;
3093 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3094
3095 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3096 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3097 AssertRCReturn(rc, rc);
3098 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3099 AssertRCReturn(rc, rc);
3100 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3101
3102 /*
3103 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3104 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3105 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3106 */
3107 uint32_t u32CR0Mask = 0;
3108 u32CR0Mask = X86_CR0_PE
3109 | X86_CR0_NE
3110 | X86_CR0_WP
3111 | X86_CR0_PG
3112 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3113 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3114 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3115 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3116 u32CR0Mask &= ~X86_CR0_PE;
3117 if (pVM->hm.s.fNestedPaging)
3118 u32CR0Mask &= ~X86_CR0_WP;
3119
3120 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3121 if (fInterceptNM)
3122 u32CR0Mask |= (X86_CR0_TS | X86_CR0_MP);
3123 else
3124 u32CR0Mask &= ~(X86_CR0_TS | X86_CR0_MP);
3125
3126 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3127 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3128 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3129 AssertRCReturn(rc, rc);
3130
3131 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3132 }
3133 return rc;
3134}
3135
3136
3137/**
3138 * Loads the guest control registers (CR3, CR4) into the guest-state area
3139 * in the VMCS.
3140 *
3141 * @returns VBox status code.
3142 * @param pVM Pointer to the VM.
3143 * @param pVCpu Pointer to the VMCPU.
3144 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3145 * out-of-sync. Make sure to update the required fields
3146 * before using them.
3147 *
3148 * @remarks No-long-jump zone!!!
3149 */
3150static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3151{
3152 int rc = VINF_SUCCESS;
3153 PVM pVM = pVCpu->CTX_SUFF(pVM);
3154
3155 /*
3156 * Guest CR2.
3157 * It's always loaded in the assembler code. Nothing to do here.
3158 */
3159
3160 /*
3161 * Guest CR3.
3162 */
3163 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3164 {
3165 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3166 if (pVM->hm.s.fNestedPaging)
3167 {
3168 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3169
3170 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3171 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3172 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3173 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3174
3175 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3176 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3177 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3178
3179 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3180 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3181 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3182 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3183
3184 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3185 AssertRCReturn(rc, rc);
3186 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3187
3188 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3189 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3190 {
3191 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3192 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3193 {
3194 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3195 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3196 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3197 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3198 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3199 }
3200
3201 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3202 have Unrestricted Execution to handle the guest when it's not using paging. */
3203 GCPhysGuestCR3 = pMixedCtx->cr3;
3204 }
3205 else
3206 {
3207 /*
3208 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3209 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3210 * EPT takes care of translating it to host-physical addresses.
3211 */
3212 RTGCPHYS GCPhys;
3213 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3214 Assert(PDMVmmDevHeapIsEnabled(pVM));
3215
3216 /* We obtain it here every time as the guest could have relocated this PCI region. */
3217 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3218 AssertRCReturn(rc, rc);
3219
3220 GCPhysGuestCR3 = GCPhys;
3221 }
3222
3223 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3224 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3225 }
3226 else
3227 {
3228 /* Non-nested paging case, just use the hypervisor's CR3. */
3229 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3230
3231 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3232 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3233 }
3234 AssertRCReturn(rc, rc);
3235
3236 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3237 }
3238
3239 /*
3240 * Guest CR4.
3241 */
3242 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3243 {
3244 Assert(!(pMixedCtx->cr4 >> 32));
3245 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3246
3247 /* The guest's view of its CR4 is unblemished. */
3248 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3249 AssertRCReturn(rc, rc);
3250 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3251
3252 /* Setup VT-x's view of the guest CR4. */
3253 /*
3254 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3255 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3256 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3257 */
3258 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3259 {
3260 Assert(pVM->hm.s.vmx.pRealModeTSS);
3261 Assert(PDMVmmDevHeapIsEnabled(pVM));
3262 u32GuestCR4 &= ~X86_CR4_VME;
3263 }
3264
3265 if (pVM->hm.s.fNestedPaging)
3266 {
3267 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3268 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3269 {
3270 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3271 u32GuestCR4 |= X86_CR4_PSE;
3272 /* Our identity mapping is a 32 bits page directory. */
3273 u32GuestCR4 &= ~X86_CR4_PAE;
3274 }
3275 /* else use guest CR4.*/
3276 }
3277 else
3278 {
3279 /*
3280 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3281 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3282 */
3283 switch (pVCpu->hm.s.enmShadowMode)
3284 {
3285 case PGMMODE_REAL: /* Real-mode. */
3286 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3287 case PGMMODE_32_BIT: /* 32-bit paging. */
3288 {
3289 u32GuestCR4 &= ~X86_CR4_PAE;
3290 break;
3291 }
3292
3293 case PGMMODE_PAE: /* PAE paging. */
3294 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3295 {
3296 u32GuestCR4 |= X86_CR4_PAE;
3297 break;
3298 }
3299
3300 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3301 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3302#ifdef VBOX_ENABLE_64_BITS_GUESTS
3303 break;
3304#endif
3305 default:
3306 AssertFailed();
3307 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3308 }
3309 }
3310
3311 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3312 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3313 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3314 u32GuestCR4 |= uSetCR4;
3315 u32GuestCR4 &= uZapCR4;
3316
3317 /* Write VT-x's view of the guest CR4 into the VMCS. */
3318 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3319 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3320 AssertRCReturn(rc, rc);
3321
3322 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3323 uint32_t u32CR4Mask = 0;
3324 u32CR4Mask = X86_CR4_VME
3325 | X86_CR4_PAE
3326 | X86_CR4_PGE
3327 | X86_CR4_PSE
3328 | X86_CR4_VMXE;
3329 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3330 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3331 AssertRCReturn(rc, rc);
3332
3333 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3334 }
3335 return rc;
3336}
3337
3338
3339/**
3340 * Loads the guest debug registers into the guest-state area in the VMCS.
3341 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3342 *
3343 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3344 *
3345 * @returns VBox status code.
3346 * @param pVCpu Pointer to the VMCPU.
3347 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3348 * out-of-sync. Make sure to update the required fields
3349 * before using them.
3350 *
3351 * @remarks No-long-jump zone!!!
3352 */
3353static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3354{
3355 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3356 return VINF_SUCCESS;
3357
3358#ifdef VBOX_STRICT
3359 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3360 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3361 {
3362 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3363 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3364 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3365 }
3366#endif
3367
3368 int rc;
3369 PVM pVM = pVCpu->CTX_SUFF(pVM);
3370 bool fInterceptDB = false;
3371 bool fInterceptMovDRx = false;
3372 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3373 {
3374 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3375 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3376 {
3377 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3378 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3379 AssertRCReturn(rc, rc);
3380 Assert(fInterceptDB == false);
3381 }
3382 else
3383 {
3384 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3385 pVCpu->hm.s.fClearTrapFlag = true;
3386 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RFLAGS;
3387 fInterceptDB = true;
3388 }
3389 }
3390
3391 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3392 {
3393 /*
3394 * Use the combined guest and host DRx values found in the hypervisor
3395 * register set because the debugger has breakpoints active or someone
3396 * is single stepping on the host side without a monitor trap flag.
3397 *
3398 * Note! DBGF expects a clean DR6 state before executing guest code.
3399 */
3400 if (!CPUMIsHyperDebugStateActive(pVCpu))
3401 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3402 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3403 Assert(CPUMIsHyperDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3404
3405 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3406 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3407 AssertRCReturn(rc, rc);
3408
3409 fInterceptDB = true;
3410 fInterceptMovDRx = true;
3411 }
3412 else
3413 {
3414 /*
3415 * If the guest has enabled debug registers, we need to load them prior to
3416 * executing guest code so they'll trigger at the right time.
3417 */
3418 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3419 {
3420 if (!CPUMIsGuestDebugStateActive(pVCpu))
3421 {
3422 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3423 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3424 }
3425 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3426 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3427 }
3428 /*
3429 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3430 * must intercept #DB in order to maintain a correct DR6 guest value.
3431 */
3432 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3433 {
3434 fInterceptMovDRx = true;
3435 fInterceptDB = true;
3436 }
3437
3438 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3439 AssertRCReturn(rc, rc);
3440 }
3441
3442 /*
3443 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3444 */
3445 if (fInterceptDB)
3446 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3447 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3448 {
3449#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3450 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3451#endif
3452 }
3453 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3454 AssertRCReturn(rc, rc);
3455
3456 /*
3457 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3458 */
3459 if (fInterceptMovDRx)
3460 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3461 else
3462 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3463 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3464 AssertRCReturn(rc, rc);
3465
3466 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3467 return VINF_SUCCESS;
3468}
3469
3470
3471#ifdef VBOX_STRICT
3472/**
3473 * Strict function to validate segment registers.
3474 *
3475 * @remarks ASSUMES CR0 is up to date.
3476 */
3477static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3478{
3479 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3480 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3481 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3482 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3483 && ( !CPUMIsGuestInRealModeEx(pCtx)
3484 && !CPUMIsGuestInV86ModeEx(pCtx)))
3485 {
3486 /* Protected mode checks */
3487 /* CS */
3488 Assert(pCtx->cs.Attr.n.u1Present);
3489 Assert(!(pCtx->cs.Attr.u & 0xf00));
3490 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3491 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3492 || !(pCtx->cs.Attr.n.u1Granularity));
3493 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3494 || (pCtx->cs.Attr.n.u1Granularity));
3495 /* CS cannot be loaded with NULL in protected mode. */
3496 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3497 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3498 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3499 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3500 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3501 else
3502 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3503 /* SS */
3504 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3505 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3506 if ( !(pCtx->cr0 & X86_CR0_PE)
3507 || pCtx->cs.Attr.n.u4Type == 3)
3508 {
3509 Assert(!pCtx->ss.Attr.n.u2Dpl);
3510 }
3511 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3512 {
3513 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3514 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3515 Assert(pCtx->ss.Attr.n.u1Present);
3516 Assert(!(pCtx->ss.Attr.u & 0xf00));
3517 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3518 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3519 || !(pCtx->ss.Attr.n.u1Granularity));
3520 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3521 || (pCtx->ss.Attr.n.u1Granularity));
3522 }
3523 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3524 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3525 {
3526 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3527 Assert(pCtx->ds.Attr.n.u1Present);
3528 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3529 Assert(!(pCtx->ds.Attr.u & 0xf00));
3530 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3531 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3532 || !(pCtx->ds.Attr.n.u1Granularity));
3533 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3534 || (pCtx->ds.Attr.n.u1Granularity));
3535 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3536 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3537 }
3538 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3539 {
3540 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3541 Assert(pCtx->es.Attr.n.u1Present);
3542 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3543 Assert(!(pCtx->es.Attr.u & 0xf00));
3544 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3545 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3546 || !(pCtx->es.Attr.n.u1Granularity));
3547 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3548 || (pCtx->es.Attr.n.u1Granularity));
3549 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3550 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3551 }
3552 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3553 {
3554 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3555 Assert(pCtx->fs.Attr.n.u1Present);
3556 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3557 Assert(!(pCtx->fs.Attr.u & 0xf00));
3558 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3559 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3560 || !(pCtx->fs.Attr.n.u1Granularity));
3561 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3562 || (pCtx->fs.Attr.n.u1Granularity));
3563 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3564 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3565 }
3566 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3567 {
3568 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3569 Assert(pCtx->gs.Attr.n.u1Present);
3570 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3571 Assert(!(pCtx->gs.Attr.u & 0xf00));
3572 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3573 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3574 || !(pCtx->gs.Attr.n.u1Granularity));
3575 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3576 || (pCtx->gs.Attr.n.u1Granularity));
3577 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3578 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3579 }
3580 /* 64-bit capable CPUs. */
3581# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3582 Assert(!(pCtx->cs.u64Base >> 32));
3583 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3584 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3585 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3586# endif
3587 }
3588 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3589 || ( CPUMIsGuestInRealModeEx(pCtx)
3590 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3591 {
3592 /* Real and v86 mode checks. */
3593 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3594 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3595 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3596 {
3597 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3598 }
3599 else
3600 {
3601 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3602 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3603 }
3604
3605 /* CS */
3606 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3607 Assert(pCtx->cs.u32Limit == 0xffff);
3608 Assert(u32CSAttr == 0xf3);
3609 /* SS */
3610 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3611 Assert(pCtx->ss.u32Limit == 0xffff);
3612 Assert(u32SSAttr == 0xf3);
3613 /* DS */
3614 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3615 Assert(pCtx->ds.u32Limit == 0xffff);
3616 Assert(u32DSAttr == 0xf3);
3617 /* ES */
3618 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3619 Assert(pCtx->es.u32Limit == 0xffff);
3620 Assert(u32ESAttr == 0xf3);
3621 /* FS */
3622 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3623 Assert(pCtx->fs.u32Limit == 0xffff);
3624 Assert(u32FSAttr == 0xf3);
3625 /* GS */
3626 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3627 Assert(pCtx->gs.u32Limit == 0xffff);
3628 Assert(u32GSAttr == 0xf3);
3629 /* 64-bit capable CPUs. */
3630# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3631 Assert(!(pCtx->cs.u64Base >> 32));
3632 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3633 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3634 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3635# endif
3636 }
3637}
3638#endif /* VBOX_STRICT */
3639
3640
3641/**
3642 * Writes a guest segment register into the guest-state area in the VMCS.
3643 *
3644 * @returns VBox status code.
3645 * @param pVCpu Pointer to the VMCPU.
3646 * @param idxSel Index of the selector in the VMCS.
3647 * @param idxLimit Index of the segment limit in the VMCS.
3648 * @param idxBase Index of the segment base in the VMCS.
3649 * @param idxAccess Index of the access rights of the segment in the VMCS.
3650 * @param pSelReg Pointer to the segment selector.
3651 * @param pCtx Pointer to the guest-CPU context.
3652 *
3653 * @remarks No-long-jump zone!!!
3654 */
3655static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3656 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3657{
3658 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3659 AssertRCReturn(rc, rc);
3660 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3661 AssertRCReturn(rc, rc);
3662 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3663 AssertRCReturn(rc, rc);
3664
3665 uint32_t u32Access = pSelReg->Attr.u;
3666 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3667 {
3668 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3669 u32Access = 0xf3;
3670 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3671 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3672 }
3673 else
3674 {
3675 /*
3676 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3677 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3678 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3679 * loaded in protected-mode have their attribute as 0.
3680 */
3681 if (!u32Access)
3682 u32Access = X86DESCATTR_UNUSABLE;
3683 }
3684
3685 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3686 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3687 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3688
3689 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3690 AssertRCReturn(rc, rc);
3691 return rc;
3692}
3693
3694
3695/**
3696 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3697 * into the guest-state area in the VMCS.
3698 *
3699 * @returns VBox status code.
3700 * @param pVM Pointer to the VM.
3701 * @param pVCPU Pointer to the VMCPU.
3702 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3703 * out-of-sync. Make sure to update the required fields
3704 * before using them.
3705 *
3706 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
3707 * @remarks No-long-jump zone!!!
3708 */
3709static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3710{
3711 int rc = VERR_INTERNAL_ERROR_5;
3712 PVM pVM = pVCpu->CTX_SUFF(pVM);
3713
3714 /*
3715 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
3716 */
3717 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3718 {
3719 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3720 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3721 {
3722 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3723 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3724 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3725 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3726 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3727 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3728 }
3729
3730#ifdef VBOX_WITH_REM
3731 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3732 {
3733 Assert(pVM->hm.s.vmx.pRealModeTSS);
3734 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3735 if ( pVCpu->hm.s.vmx.fWasInRealMode
3736 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3737 {
3738 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3739 in real-mode (e.g. OpenBSD 4.0) */
3740 REMFlushTBs(pVM);
3741 Log4(("Load: Switch to protected mode detected!\n"));
3742 pVCpu->hm.s.vmx.fWasInRealMode = false;
3743 }
3744 }
3745#endif
3746 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3747 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3748 AssertRCReturn(rc, rc);
3749 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3750 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3751 AssertRCReturn(rc, rc);
3752 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3753 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3754 AssertRCReturn(rc, rc);
3755 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3756 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3757 AssertRCReturn(rc, rc);
3758 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3759 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3760 AssertRCReturn(rc, rc);
3761 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3762 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3763 AssertRCReturn(rc, rc);
3764
3765 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3766 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3767#ifdef VBOX_STRICT
3768 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3769#endif
3770 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3771 }
3772
3773 /*
3774 * Guest TR.
3775 */
3776 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3777 {
3778 /*
3779 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3780 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3781 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3782 */
3783 uint16_t u16Sel = 0;
3784 uint32_t u32Limit = 0;
3785 uint64_t u64Base = 0;
3786 uint32_t u32AccessRights = 0;
3787
3788 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3789 {
3790 u16Sel = pMixedCtx->tr.Sel;
3791 u32Limit = pMixedCtx->tr.u32Limit;
3792 u64Base = pMixedCtx->tr.u64Base;
3793 u32AccessRights = pMixedCtx->tr.Attr.u;
3794 }
3795 else
3796 {
3797 Assert(pVM->hm.s.vmx.pRealModeTSS);
3798 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3799
3800 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3801 RTGCPHYS GCPhys;
3802 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3803 AssertRCReturn(rc, rc);
3804
3805 X86DESCATTR DescAttr;
3806 DescAttr.u = 0;
3807 DescAttr.n.u1Present = 1;
3808 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3809
3810 u16Sel = 0;
3811 u32Limit = HM_VTX_TSS_SIZE;
3812 u64Base = GCPhys; /* in real-mode phys = virt. */
3813 u32AccessRights = DescAttr.u;
3814 }
3815
3816 /* Validate. */
3817 Assert(!(u16Sel & RT_BIT(2)));
3818 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3819 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3820 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3821 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3822 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3823 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3824 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3825 Assert( (u32Limit & 0xfff) == 0xfff
3826 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3827 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3828 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3829
3830 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3831 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3832 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3833 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3834
3835 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3836 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3837 }
3838
3839 /*
3840 * Guest GDTR.
3841 */
3842 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3843 {
3844 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3845 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3846
3847 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3848 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3849 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3850 }
3851
3852 /*
3853 * Guest LDTR.
3854 */
3855 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3856 {
3857 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3858 uint32_t u32Access = 0;
3859 if (!pMixedCtx->ldtr.Attr.u)
3860 u32Access = X86DESCATTR_UNUSABLE;
3861 else
3862 u32Access = pMixedCtx->ldtr.Attr.u;
3863
3864 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3865 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3866 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3867 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3868
3869 /* Validate. */
3870 if (!(u32Access & X86DESCATTR_UNUSABLE))
3871 {
3872 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3873 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3874 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3875 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3876 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3877 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3878 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3879 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3880 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3881 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3882 }
3883
3884 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3885 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3886 }
3887
3888 /*
3889 * Guest IDTR.
3890 */
3891 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3892 {
3893 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3894 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3895
3896 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3897 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3898 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3899 }
3900
3901 return VINF_SUCCESS;
3902}
3903
3904
3905/**
3906 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3907 * areas. These MSRs will automatically be loaded to the host CPU on every
3908 * successful VM entry and stored from the host CPU on every successful VM exit.
3909 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
3910 *
3911 * @returns VBox status code.
3912 * @param pVCpu Pointer to the VMCPU.
3913 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3914 * out-of-sync. Make sure to update the required fields
3915 * before using them.
3916 *
3917 * @remarks No-long-jump zone!!!
3918 */
3919static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3920{
3921 AssertPtr(pVCpu);
3922 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3923
3924 /*
3925 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3926 */
3927 int rc = VINF_SUCCESS;
3928 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3929 {
3930#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3931 PVM pVM = pVCpu->CTX_SUFF(pVM);
3932 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3933 uint32_t cGuestMsrs = 0;
3934
3935 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3936 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3937 * when the guest really is in 64-bit mode. */
3938 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3939 if (fSupportsLongMode)
3940 {
3941 pGuestMsr->u32Msr = MSR_K8_LSTAR;
3942 pGuestMsr->u32Reserved = 0;
3943 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3944 pGuestMsr++; cGuestMsrs++;
3945 pGuestMsr->u32Msr = MSR_K6_STAR;
3946 pGuestMsr->u32Reserved = 0;
3947 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3948 pGuestMsr++; cGuestMsrs++;
3949 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
3950 pGuestMsr->u32Reserved = 0;
3951 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3952 pGuestMsr++; cGuestMsrs++;
3953 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
3954 pGuestMsr->u32Reserved = 0;
3955 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3956 pGuestMsr++; cGuestMsrs++;
3957 }
3958
3959 /*
3960 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3961 * load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3962 */
3963 if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3964 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3965 {
3966 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
3967 pGuestMsr->u32Reserved = 0;
3968 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3969 AssertRCReturn(rc, rc);
3970 pGuestMsr++; cGuestMsrs++;
3971 }
3972
3973 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3974 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
3975 {
3976 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3977 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
3978 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3979 }
3980
3981 /* Update the VCPU's copy of the guest MSR count. */
3982 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3983 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3984 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3985#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3986
3987 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3988 }
3989
3990 /*
3991 * Guest Sysenter MSRs.
3992 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3993 * VM-exits on WRMSRs for these MSRs.
3994 */
3995 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3996 {
3997 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3998 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3999 }
4000 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4001 {
4002 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4003 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
4004 }
4005 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4006 {
4007 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4008 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
4009 }
4010
4011 return rc;
4012}
4013
4014
4015/**
4016 * Loads the guest activity state into the guest-state area in the VMCS.
4017 *
4018 * @returns VBox status code.
4019 * @param pVCpu Pointer to the VMCPU.
4020 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4021 * out-of-sync. Make sure to update the required fields
4022 * before using them.
4023 *
4024 * @remarks No-long-jump zone!!!
4025 */
4026static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4027{
4028 /** @todo See if we can make use of other states, e.g.
4029 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4030 int rc = VINF_SUCCESS;
4031 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
4032 {
4033 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4034 AssertRCReturn(rc, rc);
4035 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
4036 }
4037 return rc;
4038}
4039
4040
4041/**
4042 * Sets up the appropriate function to run guest code.
4043 *
4044 * @returns VBox status code.
4045 * @param pVCpu Pointer to the VMCPU.
4046 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4047 * out-of-sync. Make sure to update the required fields
4048 * before using them.
4049 *
4050 * @remarks No-long-jump zone!!!
4051 */
4052static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4053{
4054 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4055 {
4056#ifndef VBOX_ENABLE_64_BITS_GUESTS
4057 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4058#endif
4059 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4060#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4061 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4062 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4063 {
4064 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4065 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS;
4066 }
4067#else
4068 /* 64-bit host or hybrid host. */
4069 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4070#endif
4071 }
4072 else
4073 {
4074 /* Guest is not in long mode, use the 32-bit handler. */
4075#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4076 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4077 {
4078 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4079 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_EXIT_CTLS | HM_CHANGED_VMX_ENTRY_CTLS;
4080 }
4081#else
4082 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4083#endif
4084 }
4085 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4086 return VINF_SUCCESS;
4087}
4088
4089
4090/**
4091 * Wrapper for running the guest code in VT-x.
4092 *
4093 * @returns VBox strict status code.
4094 * @param pVM Pointer to the VM.
4095 * @param pVCpu Pointer to the VMCPU.
4096 * @param pCtx Pointer to the guest-CPU context.
4097 *
4098 * @remarks No-long-jump zone!!!
4099 */
4100DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4101{
4102 /*
4103 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4104 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4105 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4106 */
4107 const bool fResumeVM = !!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4108 /** @todo Add stats for resume vs launch. */
4109#ifdef VBOX_WITH_KERNEL_USING_XMM
4110 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4111#else
4112 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4113#endif
4114}
4115
4116
4117/**
4118 * Reports world-switch error and dumps some useful debug info.
4119 *
4120 * @param pVM Pointer to the VM.
4121 * @param pVCpu Pointer to the VMCPU.
4122 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4123 * @param pCtx Pointer to the guest-CPU context.
4124 * @param pVmxTransient Pointer to the VMX transient structure (only
4125 * exitReason updated).
4126 */
4127static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4128{
4129 Assert(pVM);
4130 Assert(pVCpu);
4131 Assert(pCtx);
4132 Assert(pVmxTransient);
4133 HMVMX_ASSERT_PREEMPT_SAFE();
4134
4135 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4136 switch (rcVMRun)
4137 {
4138 case VERR_VMX_INVALID_VMXON_PTR:
4139 AssertFailed();
4140 break;
4141 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4142 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4143 {
4144 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4145 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4146 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4147 AssertRC(rc);
4148
4149 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4150 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4151 Cannot do it here as we may have been long preempted. */
4152
4153#ifdef VBOX_STRICT
4154 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4155 pVmxTransient->uExitReason));
4156 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4157 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4158 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4159 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4160 else
4161 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4162 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4163 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4164
4165 /* VMX control bits. */
4166 uint32_t u32Val;
4167 uint64_t u64Val;
4168 HMVMXHCUINTREG uHCReg;
4169 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4170 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4171 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4172 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4173 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4174 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4175 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4176 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4177 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4178 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4179 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4180 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4181 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4182 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4183 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4184 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4185 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4186 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4187 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4188 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4189 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4190 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4191 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4192 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4193 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4194 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4195 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4196 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4197 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4198 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4199 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4200 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4201 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4202 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4203 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4204 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4205 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4206 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4207 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4208 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4209 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4210 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4211
4212 /* Guest bits. */
4213 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4214 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4215 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4216 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4217 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4218 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4219 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4220 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4221
4222 /* Host bits. */
4223 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4224 Log4(("Host CR0 %#RHr\n", uHCReg));
4225 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4226 Log4(("Host CR3 %#RHr\n", uHCReg));
4227 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4228 Log4(("Host CR4 %#RHr\n", uHCReg));
4229
4230 RTGDTR HostGdtr;
4231 PCX86DESCHC pDesc;
4232 ASMGetGDTR(&HostGdtr);
4233 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4234 Log4(("Host CS %#08x\n", u32Val));
4235 if (u32Val < HostGdtr.cbGdt)
4236 {
4237 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4238 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4239 }
4240
4241 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4242 Log4(("Host DS %#08x\n", u32Val));
4243 if (u32Val < HostGdtr.cbGdt)
4244 {
4245 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4246 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4247 }
4248
4249 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4250 Log4(("Host ES %#08x\n", u32Val));
4251 if (u32Val < HostGdtr.cbGdt)
4252 {
4253 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4254 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4255 }
4256
4257 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4258 Log4(("Host FS %#08x\n", u32Val));
4259 if (u32Val < HostGdtr.cbGdt)
4260 {
4261 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4262 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4263 }
4264
4265 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4266 Log4(("Host GS %#08x\n", u32Val));
4267 if (u32Val < HostGdtr.cbGdt)
4268 {
4269 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4270 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4271 }
4272
4273 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4274 Log4(("Host SS %#08x\n", u32Val));
4275 if (u32Val < HostGdtr.cbGdt)
4276 {
4277 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4278 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4279 }
4280
4281 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4282 Log4(("Host TR %#08x\n", u32Val));
4283 if (u32Val < HostGdtr.cbGdt)
4284 {
4285 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4286 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4287 }
4288
4289 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4290 Log4(("Host TR Base %#RHv\n", uHCReg));
4291 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4292 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4293 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4294 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4295 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4296 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4297 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4298 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4299 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4300 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4301 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4302 Log4(("Host RSP %#RHv\n", uHCReg));
4303 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4304 Log4(("Host RIP %#RHv\n", uHCReg));
4305# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4306 if (HMVMX_IS_64BIT_HOST_MODE())
4307 {
4308 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4309 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4310 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4311 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4312 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4313 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4314 }
4315# endif
4316#endif /* VBOX_STRICT */
4317 break;
4318 }
4319
4320 default:
4321 /* Impossible */
4322 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4323 break;
4324 }
4325 NOREF(pVM);
4326}
4327
4328
4329#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4330#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4331# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4332#endif
4333#ifdef VBOX_STRICT
4334static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4335{
4336 switch (idxField)
4337 {
4338 case VMX_VMCS_GUEST_RIP:
4339 case VMX_VMCS_GUEST_RSP:
4340 case VMX_VMCS_GUEST_SYSENTER_EIP:
4341 case VMX_VMCS_GUEST_SYSENTER_ESP:
4342 case VMX_VMCS_GUEST_GDTR_BASE:
4343 case VMX_VMCS_GUEST_IDTR_BASE:
4344 case VMX_VMCS_GUEST_CS_BASE:
4345 case VMX_VMCS_GUEST_DS_BASE:
4346 case VMX_VMCS_GUEST_ES_BASE:
4347 case VMX_VMCS_GUEST_FS_BASE:
4348 case VMX_VMCS_GUEST_GS_BASE:
4349 case VMX_VMCS_GUEST_SS_BASE:
4350 case VMX_VMCS_GUEST_LDTR_BASE:
4351 case VMX_VMCS_GUEST_TR_BASE:
4352 case VMX_VMCS_GUEST_CR3:
4353 return true;
4354 }
4355 return false;
4356}
4357
4358static bool hmR0VmxIsValidReadField(uint32_t idxField)
4359{
4360 switch (idxField)
4361 {
4362 /* Read-only fields. */
4363 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4364 return true;
4365 }
4366 /* Remaining readable fields should also be writable. */
4367 return hmR0VmxIsValidWriteField(idxField);
4368}
4369#endif /* VBOX_STRICT */
4370
4371
4372/**
4373 * Executes the specified handler in 64-bit mode.
4374 *
4375 * @returns VBox status code.
4376 * @param pVM Pointer to the VM.
4377 * @param pVCpu Pointer to the VMCPU.
4378 * @param pCtx Pointer to the guest CPU context.
4379 * @param enmOp The operation to perform.
4380 * @param cbParam Number of parameters.
4381 * @param paParam Array of 32-bit parameters.
4382 */
4383VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4384 uint32_t *paParam)
4385{
4386 int rc, rc2;
4387 PHMGLOBALCPUINFO pCpu;
4388 RTHCPHYS HCPhysCpuPage;
4389 RTCCUINTREG uOldEflags;
4390
4391 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4392 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4393 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4394 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4395
4396#ifdef VBOX_STRICT
4397 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4398 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4399
4400 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4401 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4402#endif
4403
4404 /* Disable interrupts. */
4405 uOldEflags = ASMIntDisableFlags();
4406
4407#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4408 RTCPUID idHostCpu = RTMpCpuId();
4409 CPUMR0SetLApic(pVCpu, idHostCpu);
4410#endif
4411
4412 pCpu = HMR0GetCurrentCpu();
4413 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4414
4415 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4416 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4417
4418 /* Leave VMX Root Mode. */
4419 VMXDisable();
4420
4421 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4422
4423 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4424 CPUMSetHyperEIP(pVCpu, enmOp);
4425 for (int i = (int)cbParam - 1; i >= 0; i--)
4426 CPUMPushHyper(pVCpu, paParam[i]);
4427
4428 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4429
4430 /* Call the switcher. */
4431 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4432 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4433
4434 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4435 /* Make sure the VMX instructions don't cause #UD faults. */
4436 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4437
4438 /* Re-enter VMX Root Mode */
4439 rc2 = VMXEnable(HCPhysCpuPage);
4440 if (RT_FAILURE(rc2))
4441 {
4442 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4443 ASMSetFlags(uOldEflags);
4444 return rc2;
4445 }
4446
4447 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4448 AssertRC(rc2);
4449 Assert(!(ASMGetFlags() & X86_EFL_IF));
4450 ASMSetFlags(uOldEflags);
4451 return rc;
4452}
4453
4454
4455/**
4456 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4457 * supporting 64-bit guests.
4458 *
4459 * @returns VBox status code.
4460 * @param fResume Whether to VMLAUNCH or VMRESUME.
4461 * @param pCtx Pointer to the guest-CPU context.
4462 * @param pCache Pointer to the VMCS cache.
4463 * @param pVM Pointer to the VM.
4464 * @param pVCpu Pointer to the VMCPU.
4465 */
4466DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4467{
4468 uint32_t aParam[6];
4469 PHMGLOBALCPUINFO pCpu = NULL;
4470 RTHCPHYS HCPhysCpuPage = 0;
4471 int rc = VERR_INTERNAL_ERROR_5;
4472
4473 pCpu = HMR0GetCurrentCpu();
4474 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4475
4476#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4477 pCache->uPos = 1;
4478 pCache->interPD = PGMGetInterPaeCR3(pVM);
4479 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4480#endif
4481
4482#ifdef VBOX_STRICT
4483 pCache->TestIn.HCPhysCpuPage = 0;
4484 pCache->TestIn.HCPhysVmcs = 0;
4485 pCache->TestIn.pCache = 0;
4486 pCache->TestOut.HCPhysVmcs = 0;
4487 pCache->TestOut.pCache = 0;
4488 pCache->TestOut.pCtx = 0;
4489 pCache->TestOut.eflags = 0;
4490#endif
4491
4492 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4493 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4494 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4495 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4496 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4497 aParam[5] = 0;
4498
4499#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4500 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4501 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4502#endif
4503 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4504
4505#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4506 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4507 Assert(pCtx->dr[4] == 10);
4508 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4509#endif
4510
4511#ifdef VBOX_STRICT
4512 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4513 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4514 pVCpu->hm.s.vmx.HCPhysVmcs));
4515 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4516 pCache->TestOut.HCPhysVmcs));
4517 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4518 pCache->TestOut.pCache));
4519 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4520 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4521 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4522 pCache->TestOut.pCtx));
4523 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4524#endif
4525 return rc;
4526}
4527
4528
4529/**
4530 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4531 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4532 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4533 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4534 *
4535 * @returns VBox status code.
4536 * @param pVM Pointer to the VM.
4537 * @param pVCpu Pointer to the VMCPU.
4538 */
4539static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4540{
4541#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4542{ \
4543 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4544 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4545 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4546 ++cReadFields; \
4547}
4548
4549 AssertPtr(pVM);
4550 AssertPtr(pVCpu);
4551 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4552 uint32_t cReadFields = 0;
4553
4554 /*
4555 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4556 * and serve to indicate exceptions to the rules.
4557 */
4558
4559 /* Guest-natural selector base fields. */
4560#if 0
4561 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4562 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4563 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4564#endif
4565 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4566 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4567 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4568 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4569 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4570 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4571 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4572 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4573 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4574 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4575 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4576 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4577#if 0
4578 /* Unused natural width guest-state fields. */
4579 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4580 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4581#endif
4582 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4583 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4584
4585 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4586#if 0
4587 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4588 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4589 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4590 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4591 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4592 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4593 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4594 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4595 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4596#endif
4597
4598 /* Natural width guest-state fields. */
4599 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4600#if 0
4601 /* Currently unused field. */
4602 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4603#endif
4604
4605 if (pVM->hm.s.fNestedPaging)
4606 {
4607 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4608 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4609 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4610 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4611 }
4612 else
4613 {
4614 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4615 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4616 }
4617
4618#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4619 return VINF_SUCCESS;
4620}
4621
4622
4623/**
4624 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4625 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4626 * darwin, running 64-bit guests).
4627 *
4628 * @returns VBox status code.
4629 * @param pVCpu Pointer to the VMCPU.
4630 * @param idxField The VMCS field encoding.
4631 * @param u64Val 16, 32 or 64 bits value.
4632 */
4633VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4634{
4635 int rc;
4636 switch (idxField)
4637 {
4638 /*
4639 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4640 */
4641 /* 64-bit Control fields. */
4642 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4643 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4644 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4645 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4646 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4647 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4648 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4649 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4650 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4651 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4652 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4653 case VMX_VMCS64_CTRL_EPTP_FULL:
4654 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4655 /* 64-bit Guest-state fields. */
4656 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4657 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4658 case VMX_VMCS64_GUEST_PAT_FULL:
4659 case VMX_VMCS64_GUEST_EFER_FULL:
4660 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4661 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4662 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4663 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4664 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4665 /* 64-bit Host-state fields. */
4666 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4667 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4668 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4669 {
4670 rc = VMXWriteVmcs32(idxField, u64Val);
4671 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4672 break;
4673 }
4674
4675 /*
4676 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4677 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4678 */
4679 /* Natural-width Guest-state fields. */
4680 case VMX_VMCS_GUEST_CR3:
4681 case VMX_VMCS_GUEST_ES_BASE:
4682 case VMX_VMCS_GUEST_CS_BASE:
4683 case VMX_VMCS_GUEST_SS_BASE:
4684 case VMX_VMCS_GUEST_DS_BASE:
4685 case VMX_VMCS_GUEST_FS_BASE:
4686 case VMX_VMCS_GUEST_GS_BASE:
4687 case VMX_VMCS_GUEST_LDTR_BASE:
4688 case VMX_VMCS_GUEST_TR_BASE:
4689 case VMX_VMCS_GUEST_GDTR_BASE:
4690 case VMX_VMCS_GUEST_IDTR_BASE:
4691 case VMX_VMCS_GUEST_RSP:
4692 case VMX_VMCS_GUEST_RIP:
4693 case VMX_VMCS_GUEST_SYSENTER_ESP:
4694 case VMX_VMCS_GUEST_SYSENTER_EIP:
4695 {
4696 if (!(u64Val >> 32))
4697 {
4698 /* If this field is 64-bit, VT-x will zero out the top bits. */
4699 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4700 }
4701 else
4702 {
4703 /* Assert that only the 32->64 switcher case should ever come here. */
4704 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4705 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4706 }
4707 break;
4708 }
4709
4710 default:
4711 {
4712 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4713 rc = VERR_INVALID_PARAMETER;
4714 break;
4715 }
4716 }
4717 AssertRCReturn(rc, rc);
4718 return rc;
4719}
4720
4721
4722/**
4723 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4724 * hosts (except darwin) for 64-bit guests.
4725 *
4726 * @param pVCpu Pointer to the VMCPU.
4727 * @param idxField The VMCS field encoding.
4728 * @param u64Val 16, 32 or 64 bits value.
4729 */
4730VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4731{
4732 AssertPtr(pVCpu);
4733 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4734
4735 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4736 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4737
4738 /* Make sure there are no duplicates. */
4739 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4740 {
4741 if (pCache->Write.aField[i] == idxField)
4742 {
4743 pCache->Write.aFieldVal[i] = u64Val;
4744 return VINF_SUCCESS;
4745 }
4746 }
4747
4748 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4749 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4750 pCache->Write.cValidEntries++;
4751 return VINF_SUCCESS;
4752}
4753
4754/* Enable later when the assembly code uses these as callbacks. */
4755#if 0
4756/*
4757 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4758 *
4759 * @param pVCpu Pointer to the VMCPU.
4760 * @param pCache Pointer to the VMCS cache.
4761 *
4762 * @remarks No-long-jump zone!!!
4763 */
4764VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4765{
4766 AssertPtr(pCache);
4767 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4768 {
4769 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4770 AssertRC(rc);
4771 }
4772 pCache->Write.cValidEntries = 0;
4773}
4774
4775
4776/**
4777 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
4778 *
4779 * @param pVCpu Pointer to the VMCPU.
4780 * @param pCache Pointer to the VMCS cache.
4781 *
4782 * @remarks No-long-jump zone!!!
4783 */
4784VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4785{
4786 AssertPtr(pCache);
4787 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4788 {
4789 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4790 AssertRC(rc);
4791 }
4792}
4793#endif
4794#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4795
4796
4797/**
4798 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4799 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4800 * timer.
4801 *
4802 * @returns VBox status code.
4803 * @param pVCpu Pointer to the VMCPU.
4804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4805 * out-of-sync. Make sure to update the required fields
4806 * before using them.
4807 * @remarks No-long-jump zone!!!
4808 */
4809static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4810{
4811 int rc = VERR_INTERNAL_ERROR_5;
4812 bool fOffsettedTsc = false;
4813 PVM pVM = pVCpu->CTX_SUFF(pVM);
4814 if (pVM->hm.s.vmx.fUsePreemptTimer)
4815 {
4816 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4817
4818 /* Make sure the returned values have sane upper and lower boundaries. */
4819 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4820 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4821 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4822 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4823
4824 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4825 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4826 }
4827 else
4828 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4829
4830 if (fOffsettedTsc)
4831 {
4832 uint64_t u64CurTSC = ASMReadTSC();
4833 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4834 {
4835 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4836 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4837
4838 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4839 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4840 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4841 }
4842 else
4843 {
4844 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4845 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4846 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4847 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4848 }
4849 }
4850 else
4851 {
4852 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4853 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4854 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4855 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4856 }
4857}
4858
4859
4860/**
4861 * Determines if an exception is a contributory exception. Contributory
4862 * exceptions are ones which can cause double-faults. Page-fault is
4863 * intentionally not included here as it's a conditional contributory exception.
4864 *
4865 * @returns true if the exception is contributory, false otherwise.
4866 * @param uVector The exception vector.
4867 */
4868DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4869{
4870 switch (uVector)
4871 {
4872 case X86_XCPT_GP:
4873 case X86_XCPT_SS:
4874 case X86_XCPT_NP:
4875 case X86_XCPT_TS:
4876 case X86_XCPT_DE:
4877 return true;
4878 default:
4879 break;
4880 }
4881 return false;
4882}
4883
4884
4885/**
4886 * Sets an event as a pending event to be injected into the guest.
4887 *
4888 * @param pVCpu Pointer to the VMCPU.
4889 * @param u32IntrInfo The VM-entry interruption-information field.
4890 * @param cbInstr The VM-entry instruction length in bytes (for software
4891 * interrupts, exceptions and privileged software
4892 * exceptions).
4893 * @param u32ErrCode The VM-entry exception error code.
4894 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4895 * page-fault.
4896 *
4897 * @remarks Statistics counter assumes this is a guest event being injected or
4898 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4899 * always incremented.
4900 */
4901DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4902 RTGCUINTPTR GCPtrFaultAddress)
4903{
4904 Assert(!pVCpu->hm.s.Event.fPending);
4905 pVCpu->hm.s.Event.fPending = true;
4906 pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4907 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4908 pVCpu->hm.s.Event.cbInstr = cbInstr;
4909 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4910
4911 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4912}
4913
4914
4915/**
4916 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4917 *
4918 * @param pVCpu Pointer to the VMCPU.
4919 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4920 * out-of-sync. Make sure to update the required fields
4921 * before using them.
4922 */
4923DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4924{
4925 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
4926 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4927 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4928 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
4929}
4930
4931
4932/**
4933 * Handle a condition that occurred while delivering an event through the guest
4934 * IDT.
4935 *
4936 * @returns VBox status code (informational error codes included).
4937 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
4938 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4939 * continue execution of the guest which will delivery the #DF.
4940 * @retval VINF_EM_RESET if we detected a triple-fault condition.
4941 *
4942 * @param pVCpu Pointer to the VMCPU.
4943 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4944 * out-of-sync. Make sure to update the required fields
4945 * before using them.
4946 * @param pVmxTransient Pointer to the VMX transient structure.
4947 *
4948 * @remarks No-long-jump zone!!!
4949 */
4950static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4951{
4952 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4953 AssertRCReturn(rc, rc);
4954 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4955 {
4956 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4957 AssertRCReturn(rc, rc);
4958
4959 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4960 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4961 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4962
4963 typedef enum
4964 {
4965 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4966 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4967 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4968 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4969 } VMXREFLECTXCPT;
4970
4971 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4972 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4973 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4974 {
4975 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4976 {
4977 enmReflect = VMXREFLECTXCPT_XCPT;
4978#ifdef VBOX_STRICT
4979 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4980 && uExitVector == X86_XCPT_PF)
4981 {
4982 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4983 }
4984#endif
4985 if ( uExitVector == X86_XCPT_PF
4986 && uIdtVector == X86_XCPT_PF)
4987 {
4988 pVmxTransient->fVectoringPF = true;
4989 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4990 }
4991 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4992 && hmR0VmxIsContributoryXcpt(uExitVector)
4993 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
4994 || uIdtVector == X86_XCPT_PF))
4995 {
4996 enmReflect = VMXREFLECTXCPT_DF;
4997 }
4998 else if (uIdtVector == X86_XCPT_DF)
4999 enmReflect = VMXREFLECTXCPT_TF;
5000 }
5001 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5002 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5003 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5004 {
5005 /*
5006 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5007 * (whatever they are) as they reoccur when restarting the instruction.
5008 */
5009 enmReflect = VMXREFLECTXCPT_XCPT;
5010 }
5011 }
5012 else
5013 {
5014 /*
5015 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5016 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5017 * original exception to the guest after handling the VM-exit.
5018 */
5019 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5020 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5021 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5022 {
5023 enmReflect = VMXREFLECTXCPT_XCPT;
5024 }
5025 }
5026
5027 switch (enmReflect)
5028 {
5029 case VMXREFLECTXCPT_XCPT:
5030 {
5031 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5032 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5033 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5034
5035 uint32_t u32ErrCode = 0;
5036 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5037 {
5038 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5039 AssertRCReturn(rc, rc);
5040 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5041 }
5042
5043 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5044 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5045 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5046 rc = VINF_SUCCESS;
5047 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5048 pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
5049
5050 break;
5051 }
5052
5053 case VMXREFLECTXCPT_DF:
5054 {
5055 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5056 rc = VINF_HM_DOUBLE_FAULT;
5057 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5058 pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
5059
5060 break;
5061 }
5062
5063 case VMXREFLECTXCPT_TF:
5064 {
5065 rc = VINF_EM_RESET;
5066 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5067 uExitVector));
5068 break;
5069 }
5070
5071 default:
5072 Assert(rc == VINF_SUCCESS);
5073 break;
5074 }
5075 }
5076 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5077 return rc;
5078}
5079
5080
5081/**
5082 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5083 *
5084 * @returns VBox status code.
5085 * @param pVCpu Pointer to the VMCPU.
5086 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5087 * out-of-sync. Make sure to update the required fields
5088 * before using them.
5089 *
5090 * @remarks No-long-jump zone!!!
5091 */
5092static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5093{
5094 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5095 {
5096 uint32_t uVal = 0;
5097 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5098 AssertRCReturn(rc, rc);
5099 uint32_t uShadow = 0;
5100 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5101 AssertRCReturn(rc, rc);
5102
5103 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5104 CPUMSetGuestCR0(pVCpu, uVal);
5105 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5106 }
5107 return VINF_SUCCESS;
5108}
5109
5110
5111/**
5112 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5113 *
5114 * @returns VBox status code.
5115 * @param pVCpu Pointer to the VMCPU.
5116 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5117 * out-of-sync. Make sure to update the required fields
5118 * before using them.
5119 *
5120 * @remarks No-long-jump zone!!!
5121 */
5122static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5123{
5124 int rc = VINF_SUCCESS;
5125 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5126 {
5127 uint32_t uVal = 0;
5128 uint32_t uShadow = 0;
5129 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5130 AssertRCReturn(rc, rc);
5131 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5132 AssertRCReturn(rc, rc);
5133
5134 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5135 CPUMSetGuestCR4(pVCpu, uVal);
5136 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5137 }
5138 return rc;
5139}
5140
5141
5142/**
5143 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5144 *
5145 * @returns VBox status code.
5146 * @param pVCpu Pointer to the VMCPU.
5147 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5148 * out-of-sync. Make sure to update the required fields
5149 * before using them.
5150 *
5151 * @remarks No-long-jump zone!!!
5152 */
5153static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5154{
5155 int rc = VINF_SUCCESS;
5156 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5157 {
5158 uint64_t u64Val = 0;
5159 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5160 AssertRCReturn(rc, rc);
5161
5162 pMixedCtx->rip = u64Val;
5163 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5164 }
5165 return rc;
5166}
5167
5168
5169/**
5170 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5171 *
5172 * @returns VBox status code.
5173 * @param pVCpu Pointer to the VMCPU.
5174 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5175 * out-of-sync. Make sure to update the required fields
5176 * before using them.
5177 *
5178 * @remarks No-long-jump zone!!!
5179 */
5180static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5181{
5182 int rc = VINF_SUCCESS;
5183 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5184 {
5185 uint64_t u64Val = 0;
5186 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5187 AssertRCReturn(rc, rc);
5188
5189 pMixedCtx->rsp = u64Val;
5190 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5191 }
5192 return rc;
5193}
5194
5195
5196/**
5197 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5198 *
5199 * @returns VBox status code.
5200 * @param pVCpu Pointer to the VMCPU.
5201 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5202 * out-of-sync. Make sure to update the required fields
5203 * before using them.
5204 *
5205 * @remarks No-long-jump zone!!!
5206 */
5207static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5208{
5209 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5210 {
5211 uint32_t uVal = 0;
5212 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5213 AssertRCReturn(rc, rc);
5214
5215 pMixedCtx->eflags.u32 = uVal;
5216 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5217 {
5218 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5219 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5220
5221 pMixedCtx->eflags.Bits.u1VM = 0;
5222 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5223 }
5224
5225 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5226 }
5227 return VINF_SUCCESS;
5228}
5229
5230
5231/**
5232 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5233 * guest-CPU context.
5234 */
5235DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5236{
5237 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5238 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5239 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5240 return rc;
5241}
5242
5243
5244/**
5245 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5246 * from the guest-state area in the VMCS.
5247 *
5248 * @param pVCpu Pointer to the VMCPU.
5249 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5250 * out-of-sync. Make sure to update the required fields
5251 * before using them.
5252 *
5253 * @remarks No-long-jump zone!!!
5254 */
5255static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5256{
5257 uint32_t uIntrState = 0;
5258 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5259 AssertRC(rc);
5260
5261 if (!uIntrState)
5262 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5263 else
5264 {
5265 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5266 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5267 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5268 AssertRC(rc);
5269 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5270 AssertRC(rc);
5271
5272 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5273 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5274 }
5275}
5276
5277
5278/**
5279 * Saves the guest's activity state.
5280 *
5281 * @returns VBox status code.
5282 * @param pVCpu Pointer to the VMCPU.
5283 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5284 * out-of-sync. Make sure to update the required fields
5285 * before using them.
5286 *
5287 * @remarks No-long-jump zone!!!
5288 */
5289static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5290{
5291 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5292 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5293 return VINF_SUCCESS;
5294}
5295
5296
5297/**
5298 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5299 * the current VMCS into the guest-CPU context.
5300 *
5301 * @returns VBox status code.
5302 * @param pVCpu Pointer to the VMCPU.
5303 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5304 * out-of-sync. Make sure to update the required fields
5305 * before using them.
5306 *
5307 * @remarks No-long-jump zone!!!
5308 */
5309static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5310{
5311 int rc = VINF_SUCCESS;
5312 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5313 {
5314 uint32_t u32Val = 0;
5315 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5316 pMixedCtx->SysEnter.cs = u32Val;
5317 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5318 }
5319
5320 uint64_t u64Val = 0;
5321 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5322 {
5323 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5324 pMixedCtx->SysEnter.eip = u64Val;
5325 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5326 }
5327 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5328 {
5329 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5330 pMixedCtx->SysEnter.esp = u64Val;
5331 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5332 }
5333 return rc;
5334}
5335
5336
5337/**
5338 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5339 * context.
5340 *
5341 * @returns VBox status code.
5342 * @param pVCpu Pointer to the VMCPU.
5343 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5344 * out-of-sync. Make sure to update the required fields
5345 * before using them.
5346 *
5347 * @remarks No-long-jump zone!!!
5348 */
5349static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5350{
5351 int rc = VINF_SUCCESS;
5352 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5353 {
5354 uint64_t u64Val = 0;
5355 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5356 pMixedCtx->fs.u64Base = u64Val;
5357 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5358 }
5359 return rc;
5360}
5361
5362
5363/**
5364 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5365 * context.
5366 *
5367 * @returns VBox status code.
5368 * @param pVCpu Pointer to the VMCPU.
5369 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5370 * out-of-sync. Make sure to update the required fields
5371 * before using them.
5372 *
5373 * @remarks No-long-jump zone!!!
5374 */
5375static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5376{
5377 int rc = VINF_SUCCESS;
5378 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5379 {
5380 uint64_t u64Val = 0;
5381 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5382 pMixedCtx->gs.u64Base = u64Val;
5383 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5384 }
5385 return rc;
5386}
5387
5388
5389/**
5390 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5391 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5392 * and TSC_AUX.
5393 *
5394 * @returns VBox status code.
5395 * @param pVCpu Pointer to the VMCPU.
5396 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5397 * out-of-sync. Make sure to update the required fields
5398 * before using them.
5399 *
5400 * @remarks No-long-jump zone!!!
5401 */
5402static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5403{
5404 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5405 return VINF_SUCCESS;
5406
5407#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5408 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5409 {
5410 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5411 pMsr += i;
5412 switch (pMsr->u32Msr)
5413 {
5414 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5415 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5416 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5417 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5418 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5419 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5420 default:
5421 {
5422 AssertFailed();
5423 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5424 }
5425 }
5426 }
5427#endif
5428
5429 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5430 return VINF_SUCCESS;
5431}
5432
5433
5434/**
5435 * Saves the guest control registers from the current VMCS into the guest-CPU
5436 * context.
5437 *
5438 * @returns VBox status code.
5439 * @param pVCpu Pointer to the VMCPU.
5440 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5441 * out-of-sync. Make sure to update the required fields
5442 * before using them.
5443 *
5444 * @remarks No-long-jump zone!!!
5445 */
5446static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5447{
5448 /* Guest CR0. Guest FPU. */
5449 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5450 AssertRCReturn(rc, rc);
5451
5452 /* Guest CR4. */
5453 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5454 AssertRCReturn(rc, rc);
5455
5456 /* Guest CR2 - updated always during the world-switch or in #PF. */
5457 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5458 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5459 {
5460 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5461 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5462
5463 PVM pVM = pVCpu->CTX_SUFF(pVM);
5464 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5465 || ( pVM->hm.s.fNestedPaging
5466 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5467 {
5468 uint64_t u64Val = 0;
5469 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5470 if (pMixedCtx->cr3 != u64Val)
5471 {
5472 CPUMSetGuestCR3(pVCpu, u64Val);
5473 if (VMMRZCallRing3IsEnabled(pVCpu))
5474 {
5475 PGMUpdateCR3(pVCpu, u64Val);
5476 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5477 }
5478 else
5479 {
5480 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5481 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5482 }
5483 }
5484
5485 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5486 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5487 {
5488 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5489 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5490 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5491 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5492
5493 if (VMMRZCallRing3IsEnabled(pVCpu))
5494 {
5495 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5496 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5497 }
5498 else
5499 {
5500 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5501 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5502 }
5503 }
5504 }
5505
5506 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5507 }
5508
5509 /*
5510 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5511 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5512 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5513 *
5514 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5515 */
5516 if (VMMRZCallRing3IsEnabled(pVCpu))
5517 {
5518 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5519 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5520
5521 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5522 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5523
5524 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5525 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5526 }
5527
5528 return rc;
5529}
5530
5531
5532/**
5533 * Reads a guest segment register from the current VMCS into the guest-CPU
5534 * context.
5535 *
5536 * @returns VBox status code.
5537 * @param pVCpu Pointer to the VMCPU.
5538 * @param idxSel Index of the selector in the VMCS.
5539 * @param idxLimit Index of the segment limit in the VMCS.
5540 * @param idxBase Index of the segment base in the VMCS.
5541 * @param idxAccess Index of the access rights of the segment in the VMCS.
5542 * @param pSelReg Pointer to the segment selector.
5543 *
5544 * @remarks No-long-jump zone!!!
5545 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5546 * macro as that takes care of whether to read from the VMCS cache or
5547 * not.
5548 */
5549DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5550 PCPUMSELREG pSelReg)
5551{
5552 uint32_t u32Val = 0;
5553 int rc = VMXReadVmcs32(idxSel, &u32Val);
5554 AssertRCReturn(rc, rc);
5555 pSelReg->Sel = (uint16_t)u32Val;
5556 pSelReg->ValidSel = (uint16_t)u32Val;
5557 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5558
5559 rc = VMXReadVmcs32(idxLimit, &u32Val);
5560 AssertRCReturn(rc, rc);
5561 pSelReg->u32Limit = u32Val;
5562
5563 uint64_t u64Val = 0;
5564 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5565 AssertRCReturn(rc, rc);
5566 pSelReg->u64Base = u64Val;
5567
5568 rc = VMXReadVmcs32(idxAccess, &u32Val);
5569 AssertRCReturn(rc, rc);
5570 pSelReg->Attr.u = u32Val;
5571
5572 /*
5573 * If VT-x marks the segment as unusable, most other bits remain undefined:
5574 * - For CS the L, D and G bits have meaning.
5575 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5576 * - For the remaining data segments no bits are defined.
5577 *
5578 * The present bit and the unusable bit has been observed to be set at the
5579 * same time (the selector was supposed to invalid as we started executing
5580 * a V8086 interrupt in ring-0).
5581 *
5582 * What should be important for the rest of the VBox code that the P bit is
5583 * cleared. Some of the other VBox code recognizes the unusable bit, but
5584 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5585 * safe side here, we'll strip off P and other bits we don't care about. If
5586 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5587 *
5588 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5589 */
5590 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5591 {
5592 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5593
5594 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5595 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5596 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5597
5598 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5599#ifdef DEBUG_bird
5600 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5601 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5602 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5603#endif
5604 }
5605 return VINF_SUCCESS;
5606}
5607
5608
5609#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5610# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5611 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5612 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5613#else
5614# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5615 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5616 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5617#endif
5618
5619
5620/**
5621 * Saves the guest segment registers from the current VMCS into the guest-CPU
5622 * context.
5623 *
5624 * @returns VBox status code.
5625 * @param pVCpu Pointer to the VMCPU.
5626 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5627 * out-of-sync. Make sure to update the required fields
5628 * before using them.
5629 *
5630 * @remarks No-long-jump zone!!!
5631 */
5632static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5633{
5634 /* Guest segment registers. */
5635 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5636 {
5637 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5638 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5639 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5640 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5641 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5642 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5643 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5644
5645 /* Restore segment attributes for real-on-v86 mode hack. */
5646 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5647 {
5648 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5649 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5650 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5651 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5652 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5653 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5654 }
5655 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5656 }
5657
5658 return VINF_SUCCESS;
5659}
5660
5661
5662/**
5663 * Saves the guest descriptor table registers and task register from the current
5664 * VMCS into the guest-CPU context.
5665 *
5666 * @returns VBox status code.
5667 * @param pVCpu Pointer to the VMCPU.
5668 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5669 * out-of-sync. Make sure to update the required fields
5670 * before using them.
5671 *
5672 * @remarks No-long-jump zone!!!
5673 */
5674static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5675{
5676 int rc = VINF_SUCCESS;
5677
5678 /* Guest LDTR. */
5679 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5680 {
5681 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5682 AssertRCReturn(rc, rc);
5683 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5684 }
5685
5686 /* Guest GDTR. */
5687 uint64_t u64Val = 0;
5688 uint32_t u32Val = 0;
5689 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5690 {
5691 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5692 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5693 pMixedCtx->gdtr.pGdt = u64Val;
5694 pMixedCtx->gdtr.cbGdt = u32Val;
5695 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
5696 }
5697
5698 /* Guest IDTR. */
5699 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5700 {
5701 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5702 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5703 pMixedCtx->idtr.pIdt = u64Val;
5704 pMixedCtx->idtr.cbIdt = u32Val;
5705 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
5706 }
5707
5708 /* Guest TR. */
5709 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5710 {
5711 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5712 AssertRCReturn(rc, rc);
5713
5714 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5715 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5716 {
5717 rc = VMXLOCAL_READ_SEG(TR, tr);
5718 AssertRCReturn(rc, rc);
5719 }
5720 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
5721 }
5722 return rc;
5723}
5724
5725#undef VMXLOCAL_READ_SEG
5726
5727
5728/**
5729 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5730 * context.
5731 *
5732 * @returns VBox status code.
5733 * @param pVCpu Pointer to the VMCPU.
5734 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5735 * out-of-sync. Make sure to update the required fields
5736 * before using them.
5737 *
5738 * @remarks No-long-jump zone!!!
5739 */
5740static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5741{
5742 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5743 {
5744 if (!CPUMIsHyperDebugStateActive(pVCpu))
5745 {
5746 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5747 uint32_t u32Val;
5748 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5749 pMixedCtx->dr[7] = u32Val;
5750 }
5751
5752 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
5753 }
5754 return VINF_SUCCESS;
5755}
5756
5757
5758/**
5759 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
5760 *
5761 * @returns VBox status code.
5762 * @param pVCpu Pointer to the VMCPU.
5763 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5764 * out-of-sync. Make sure to update the required fields
5765 * before using them.
5766 *
5767 * @remarks No-long-jump zone!!!
5768 */
5769static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5770{
5771 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5772 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
5773 return VINF_SUCCESS;
5774}
5775
5776
5777/**
5778 * Saves the entire guest state from the currently active VMCS into the
5779 * guest-CPU context. This essentially VMREADs all guest-data.
5780 *
5781 * @returns VBox status code.
5782 * @param pVCpu Pointer to the VMCPU.
5783 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5784 * out-of-sync. Make sure to update the required fields
5785 * before using them.
5786 */
5787static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5788{
5789 Assert(pVCpu);
5790 Assert(pMixedCtx);
5791
5792 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5793 return VINF_SUCCESS;
5794
5795 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5796 again on the ring-3 callback path, there is no real need to. */
5797 if (VMMRZCallRing3IsEnabled(pVCpu))
5798 VMMR0LogFlushDisable(pVCpu);
5799 else
5800 Assert(VMMR0IsLogFlushDisabled(pVCpu));
5801 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5802
5803 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5804 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5805
5806 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5807 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5808
5809 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5810 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5811
5812 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5813 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5814
5815 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5816 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5817
5818 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5819 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5820
5821 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5822 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5823
5824 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5825 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5826
5827 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5828 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5829
5830 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5831 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5832
5833 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5834 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5835
5836 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5837 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5838
5839 if (VMMRZCallRing3IsEnabled(pVCpu))
5840 VMMR0LogFlushEnable(pVCpu);
5841
5842 return rc;
5843}
5844
5845
5846/**
5847 * Check per-VM and per-VCPU force flag actions that require us to go back to
5848 * ring-3 for one reason or another.
5849 *
5850 * @returns VBox status code (information status code included).
5851 * @retval VINF_SUCCESS if we don't have any actions that require going back to
5852 * ring-3.
5853 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5854 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5855 * interrupts)
5856 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5857 * all EMTs to be in ring-3.
5858 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5859 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5860 * to the EM loop.
5861 *
5862 * @param pVM Pointer to the VM.
5863 * @param pVCpu Pointer to the VMCPU.
5864 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5865 * out-of-sync. Make sure to update the required fields
5866 * before using them.
5867 */
5868static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5869{
5870 Assert(VMMRZCallRing3IsEnabled(pVCpu));
5871
5872 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5873 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5874 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5875 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5876 {
5877 /* We need the control registers now, make sure the guest-CPU context is updated. */
5878 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5879 AssertRCReturn(rc3, rc3);
5880
5881 /* Pending HM CR3 sync. */
5882 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5883 {
5884 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5885 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
5886 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5887 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5888 }
5889
5890 /* Pending HM PAE PDPEs. */
5891 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5892 {
5893 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5894 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5895 }
5896
5897 /* Pending PGM C3 sync. */
5898 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5899 {
5900 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
5901 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5902 if (rc2 != VINF_SUCCESS)
5903 {
5904 AssertRC(rc2);
5905 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5906 return rc2;
5907 }
5908 }
5909
5910 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5911 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5912 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5913 {
5914 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5915 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5916 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5917 return rc2;
5918 }
5919
5920 /* Pending VM request packets, such as hardware interrupts. */
5921 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5922 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5923 {
5924 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5925 return VINF_EM_PENDING_REQUEST;
5926 }
5927
5928 /* Pending PGM pool flushes. */
5929 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5930 {
5931 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5932 return VINF_PGM_POOL_FLUSH_PENDING;
5933 }
5934
5935 /* Pending DMA requests. */
5936 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5937 {
5938 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5939 return VINF_EM_RAW_TO_R3;
5940 }
5941 }
5942
5943 return VINF_SUCCESS;
5944}
5945
5946
5947/**
5948 * Converts any TRPM trap into a pending HM event. This is typically used when
5949 * entering from ring-3 (not longjmp returns).
5950 *
5951 * @param pVCpu Pointer to the VMCPU.
5952 */
5953static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5954{
5955 Assert(TRPMHasTrap(pVCpu));
5956 Assert(!pVCpu->hm.s.Event.fPending);
5957
5958 uint8_t uVector;
5959 TRPMEVENT enmTrpmEvent;
5960 RTGCUINT uErrCode;
5961 RTGCUINTPTR GCPtrFaultAddress;
5962 uint8_t cbInstr;
5963
5964 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5965 AssertRC(rc);
5966
5967 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5968 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
5969 if (enmTrpmEvent == TRPM_TRAP)
5970 {
5971 switch (uVector)
5972 {
5973 case X86_XCPT_BP:
5974 case X86_XCPT_OF:
5975 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5976 break;
5977
5978 case X86_XCPT_PF:
5979 case X86_XCPT_DF:
5980 case X86_XCPT_TS:
5981 case X86_XCPT_NP:
5982 case X86_XCPT_SS:
5983 case X86_XCPT_GP:
5984 case X86_XCPT_AC:
5985 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5986 /* no break! */
5987 default:
5988 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5989 break;
5990 }
5991 }
5992 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5993 {
5994 if (uVector == X86_XCPT_NMI)
5995 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5996 else
5997 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5998 }
5999 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6000 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6001 else
6002 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6003
6004 rc = TRPMResetTrap(pVCpu);
6005 AssertRC(rc);
6006 Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6007 u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6008
6009 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6010 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6011}
6012
6013
6014/**
6015 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6016 * VT-x to execute any instruction.
6017 *
6018 * @param pvCpu Pointer to the VMCPU.
6019 */
6020static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6021{
6022 Assert(pVCpu->hm.s.Event.fPending);
6023
6024 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6025 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
6026 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
6027 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6028
6029 /* If a trap was already pending, we did something wrong! */
6030 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6031
6032 TRPMEVENT enmTrapType;
6033 switch (uVectorType)
6034 {
6035 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6036 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6037 enmTrapType = TRPM_HARDWARE_INT;
6038 break;
6039
6040 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6041 enmTrapType = TRPM_SOFTWARE_INT;
6042 break;
6043
6044 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6045 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6046 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6047 enmTrapType = TRPM_TRAP;
6048 break;
6049
6050 default:
6051 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6052 enmTrapType = TRPM_32BIT_HACK;
6053 break;
6054 }
6055
6056 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6057
6058 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6059 AssertRC(rc);
6060
6061 if (fErrorCodeValid)
6062 TRPMSetErrorCode(pVCpu, uErrorCode);
6063
6064 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6065 && uVector == X86_XCPT_PF)
6066 {
6067 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6068 }
6069 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6070 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6071 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6072 {
6073 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6074 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6075 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6076 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6077 }
6078 pVCpu->hm.s.Event.fPending = false;
6079}
6080
6081
6082/**
6083 * Does the necessary state syncing before returning to ring-3 for any reason
6084 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6085 *
6086 * @returns VBox status code.
6087 * @param pVM Pointer to the VM.
6088 * @param pVCpu Pointer to the VMCPU.
6089 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6090 * out-of-sync. Make sure to update the required fields
6091 * before using them.
6092 *
6093 * @remarks No-long-jmp zone!!!
6094 */
6095static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6096{
6097 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6098 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6099
6100 RTCPUID idCpu = RTMpCpuId();
6101 Log4Func(("HostCpuId=%u\n", idCpu));
6102
6103 /* Save the guest state if necessary. */
6104 if (pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6105 {
6106 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6107 AssertRCReturn(rc, rc);
6108 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6109 }
6110
6111 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6112 if (CPUMIsGuestFPUStateActive(pVCpu))
6113 {
6114 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6115 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6116 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
6117 }
6118
6119 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6120#ifdef VBOX_STRICT
6121 if (CPUMIsHyperDebugStateActive(pVCpu))
6122 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6123#endif
6124 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6125 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
6126 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6127 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6128
6129 /* Restore host-state bits that VT-x only restores partially. */
6130 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6131 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6132 {
6133 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6134 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6135 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6136 }
6137
6138 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6139 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6140 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6141 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6142 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6143 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6144 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6145 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6146
6147 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6148
6149 /** @todo This kinda defeats the purpose of having preemption hooks.
6150 * The problem is, deregistering the hooks should be moved to a place that
6151 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6152 * context.
6153 */
6154 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6155 {
6156 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6157 AssertRCReturn(rc, rc);
6158
6159 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6160 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6161 }
6162 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6163 NOREF(idCpu);
6164
6165 return VINF_SUCCESS;
6166}
6167
6168
6169/**
6170 * Leaves the VT-x session.
6171 *
6172 * @returns VBox status code.
6173 * @param pVM Pointer to the VM.
6174 * @param pVCpu Pointer to the VMCPU.
6175 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6176 * out-of-sync. Make sure to update the required fields
6177 * before using them.
6178 *
6179 * @remarks No-long-jmp zone!!!
6180 */
6181DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6182{
6183 HM_DISABLE_PREEMPT_IF_NEEDED();
6184 HMVMX_ASSERT_CPU_SAFE();
6185 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6186 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6187
6188 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6189 and done this from the VMXR0ThreadCtxCallback(). */
6190 if (!pVCpu->hm.s.fLeaveDone)
6191 {
6192 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6193 AssertRCReturn(rc2, rc2);
6194 pVCpu->hm.s.fLeaveDone = true;
6195 }
6196
6197 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6198 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6199 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6200 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6201 VMMR0ThreadCtxHooksDeregister(pVCpu);
6202
6203 /* Leave HM context. This takes care of local init (term). */
6204 int rc = HMR0LeaveCpu(pVCpu);
6205
6206 HM_RESTORE_PREEMPT_IF_NEEDED();
6207
6208 return rc;
6209}
6210
6211
6212/**
6213 * Does the necessary state syncing before doing a longjmp to ring-3.
6214 *
6215 * @returns VBox status code.
6216 * @param pVM Pointer to the VM.
6217 * @param pVCpu Pointer to the VMCPU.
6218 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6219 * out-of-sync. Make sure to update the required fields
6220 * before using them.
6221 *
6222 * @remarks No-long-jmp zone!!!
6223 */
6224DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6225{
6226 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6227}
6228
6229
6230/**
6231 * Take necessary actions before going back to ring-3.
6232 *
6233 * An action requires us to go back to ring-3. This function does the necessary
6234 * steps before we can safely return to ring-3. This is not the same as longjmps
6235 * to ring-3, this is voluntary and prepares the guest so it may continue
6236 * executing outside HM (recompiler/IEM).
6237 *
6238 * @returns VBox status code.
6239 * @param pVM Pointer to the VM.
6240 * @param pVCpu Pointer to the VMCPU.
6241 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6242 * out-of-sync. Make sure to update the required fields
6243 * before using them.
6244 * @param rcExit The reason for exiting to ring-3. Can be
6245 * VINF_VMM_UNKNOWN_RING3_CALL.
6246 */
6247static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6248{
6249 Assert(pVM);
6250 Assert(pVCpu);
6251 Assert(pMixedCtx);
6252 HMVMX_ASSERT_PREEMPT_SAFE();
6253
6254 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6255 {
6256 /* We've done what is required in hmR0VmxExitErrInvalidGuestState(). We're not going to continue guest execution... */
6257 return VINF_SUCCESS;
6258 }
6259 else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6260 {
6261 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6262 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6263 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6264 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6265 return VINF_SUCCESS;
6266 }
6267
6268 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6269 VMMRZCallRing3Disable(pVCpu);
6270 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6271
6272 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6273 if (pVCpu->hm.s.Event.fPending)
6274 {
6275 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6276 Assert(!pVCpu->hm.s.Event.fPending);
6277 }
6278
6279 /* Save guest state and restore host state bits. */
6280 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6281 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6282
6283 /* Sync recompiler state. */
6284 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6285 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6286 | CPUM_CHANGED_LDTR
6287 | CPUM_CHANGED_GDTR
6288 | CPUM_CHANGED_IDTR
6289 | CPUM_CHANGED_TR
6290 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6291 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6292 if ( pVM->hm.s.fNestedPaging
6293 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6294 {
6295 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6296 }
6297
6298 /*
6299 * Clear the X86_EFL_TF if necessary.
6300 */
6301 if (pVCpu->hm.s.fClearTrapFlag)
6302 {
6303 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6304 pMixedCtx->eflags.Bits.u1TF = 0;
6305 pVCpu->hm.s.fClearTrapFlag = false;
6306 }
6307 /** @todo there seems to be issues with the resume flag when the monitor trap
6308 * flag is pending without being used. Seen early in bios init when
6309 * accessing APIC page in prot mode. */
6310
6311 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6312 if (rcExit != VINF_EM_RAW_INTERRUPT)
6313 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
6314
6315 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6316
6317 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6318 VMMRZCallRing3RemoveNotification(pVCpu);
6319 VMMRZCallRing3Enable(pVCpu);
6320
6321 return rc;
6322}
6323
6324
6325/**
6326 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6327 * longjump to ring-3 and possibly get preempted.
6328 *
6329 * @returns VBox status code.
6330 * @param pVCpu Pointer to the VMCPU.
6331 * @param enmOperation The operation causing the ring-3 longjump.
6332 * @param pvUser Opaque pointer to the guest-CPU context. The data
6333 * may be out-of-sync. Make sure to update the required
6334 * fields before using them.
6335 *
6336 * @remarks Must never be called with @a enmOperation ==
6337 * VMMCALLRING3_VM_R0_ASSERTION. We can't assert it here because if it
6338 * it -does- get called with VMMCALLRING3_VM_R0_ASSERTION, we'll end up
6339 * with an infinite recursion.
6340 */
6341DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6342{
6343 /* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6344 Assert(pVCpu);
6345 Assert(pvUser);
6346 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6347 HMVMX_ASSERT_PREEMPT_SAFE();
6348
6349 VMMRZCallRing3Disable(pVCpu);
6350 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6351
6352 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6353 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6354 AssertRCReturn(rc, rc);
6355
6356 VMMRZCallRing3Enable(pVCpu);
6357 return VINF_SUCCESS;
6358}
6359
6360
6361/**
6362 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6363 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6364 *
6365 * @param pVCpu Pointer to the VMCPU.
6366 */
6367DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6368{
6369 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6370 {
6371 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6372 {
6373 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6374 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6375 AssertRC(rc);
6376 }
6377 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6378}
6379
6380
6381/**
6382 * Evaluates the event to be delivered to the guest and sets it as the pending
6383 * event.
6384 *
6385 * @param pVCpu Pointer to the VMCPU.
6386 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6387 * out-of-sync. Make sure to update the required fields
6388 * before using them.
6389 */
6390static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6391{
6392 Assert(!pVCpu->hm.s.Event.fPending);
6393
6394 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6395 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6396 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6397 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6398
6399 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6400 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6401 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6402 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6403 Assert(!TRPMHasTrap(pVCpu));
6404
6405 /** @todo SMI. SMIs take priority over NMIs. */
6406 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6407 {
6408 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6409 if ( !fBlockMovSS
6410 && !fBlockSti)
6411 {
6412 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6413 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6414 uint32_t u32IntrInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6415 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6416
6417 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6418 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6419 }
6420 else
6421 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6422 }
6423 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6424 && !pVCpu->hm.s.fSingleInstruction)
6425 {
6426 /*
6427 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6428 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6429 * evaluated here and not set as pending, solely based on the force-flags.
6430 */
6431 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6432 AssertRC(rc);
6433 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6434 if ( !fBlockInt
6435 && !fBlockSti
6436 && !fBlockMovSS)
6437 {
6438 uint8_t u8Interrupt;
6439 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6440 if (RT_SUCCESS(rc))
6441 {
6442 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6443 uint32_t u32IntrInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6444 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6445
6446 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6447 }
6448 else
6449 {
6450 /** @todo Does this actually happen? If not turn it into an assertion. */
6451 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6452 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6453 }
6454 }
6455 else
6456 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6457 }
6458}
6459
6460
6461/**
6462 * Injects any pending events into the guest if the guest is in a state to
6463 * receive them.
6464 *
6465 * @returns VBox status code (informational status codes included).
6466 * @param pVCpu Pointer to the VMCPU.
6467 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6468 * out-of-sync. Make sure to update the required fields
6469 * before using them.
6470 */
6471static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6472{
6473 HMVMX_ASSERT_PREEMPT_SAFE();
6474 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6475
6476 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6477 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6478 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6479 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6480
6481 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6482 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6483 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6484 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6485 Assert(!TRPMHasTrap(pVCpu));
6486
6487 int rc = VINF_SUCCESS;
6488 if (pVCpu->hm.s.Event.fPending)
6489 {
6490#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6491 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6492 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6493 {
6494 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6495 AssertRCReturn(rc, rc);
6496 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6497 Assert(!fBlockInt);
6498 Assert(!fBlockSti);
6499 Assert(!fBlockMovSS);
6500 }
6501 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6502 {
6503 Assert(!fBlockSti);
6504 Assert(!fBlockMovSS);
6505 }
6506#endif
6507 Log4(("Injecting pending event vcpu[%RU32] u64IntrInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo));
6508 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6509 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6510 AssertRCReturn(rc, rc);
6511
6512 /* Update the interruptibility-state as it could have been changed by
6513 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6514 fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6515 fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6516
6517#ifdef VBOX_WITH_STATISTICS
6518 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6519 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6520 else
6521 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6522#endif
6523 }
6524
6525 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6526 int rc2 = VINF_SUCCESS;
6527 if ( fBlockSti
6528 || fBlockMovSS)
6529 {
6530 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6531 {
6532 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6533 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6534 {
6535 /*
6536 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6537 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6538 * See Intel spec. 27.3.4 "Saving Non-Register State".
6539 */
6540 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6541 AssertRCReturn(rc, rc);
6542 }
6543 }
6544 else
6545 {
6546 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6547 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6548 uIntrState = 0;
6549 }
6550 }
6551
6552 /*
6553 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6554 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6555 */
6556 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6557 AssertRC(rc2);
6558
6559 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6560 return rc;
6561}
6562
6563
6564/**
6565 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6566 *
6567 * @param pVCpu Pointer to the VMCPU.
6568 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6569 * out-of-sync. Make sure to update the required fields
6570 * before using them.
6571 */
6572DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6573{
6574 uint32_t u32IntrInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6575 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6576}
6577
6578
6579/**
6580 * Injects a double-fault (#DF) exception into the VM.
6581 *
6582 * @returns VBox status code (informational status code included).
6583 * @param pVCpu Pointer to the VMCPU.
6584 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6585 * out-of-sync. Make sure to update the required fields
6586 * before using them.
6587 */
6588DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6589{
6590 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6591 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6592 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6593 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6594 puIntrState);
6595}
6596
6597
6598/**
6599 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6600 *
6601 * @param pVCpu Pointer to the VMCPU.
6602 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6603 * out-of-sync. Make sure to update the required fields
6604 * before using them.
6605 */
6606DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6607{
6608 uint32_t u32IntrInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6609 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6610 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6611}
6612
6613
6614/**
6615 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6616 *
6617 * @param pVCpu Pointer to the VMCPU.
6618 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6619 * out-of-sync. Make sure to update the required fields
6620 * before using them.
6621 * @param cbInstr The value of RIP that is to be pushed on the guest
6622 * stack.
6623 */
6624DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6625{
6626 uint32_t u32IntrInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6627 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6628 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6629}
6630
6631
6632/**
6633 * Injects a general-protection (#GP) fault into the VM.
6634 *
6635 * @returns VBox status code (informational status code included).
6636 * @param pVCpu Pointer to the VMCPU.
6637 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6638 * out-of-sync. Make sure to update the required fields
6639 * before using them.
6640 * @param u32ErrorCode The error code associated with the #GP.
6641 */
6642DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6643 uint32_t *puIntrState)
6644{
6645 uint32_t u32IntrInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6646 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6647 if (fErrorCodeValid)
6648 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6649 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6650 puIntrState);
6651}
6652
6653
6654/**
6655 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6656 *
6657 * @param pVCpu Pointer to the VMCPU.
6658 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6659 * out-of-sync. Make sure to update the required fields
6660 * before using them.
6661 * @param uVector The software interrupt vector number.
6662 * @param cbInstr The value of RIP that is to be pushed on the guest
6663 * stack.
6664 */
6665DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6666{
6667 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6668 if ( uVector == X86_XCPT_BP
6669 || uVector == X86_XCPT_OF)
6670 {
6671 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6672 }
6673 else
6674 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6675 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6676}
6677
6678
6679/**
6680 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6681 * stack.
6682 *
6683 * @returns VBox status code (information status code included).
6684 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6685 * @param pVM Pointer to the VM.
6686 * @param pMixedCtx Pointer to the guest-CPU context.
6687 * @param uValue The value to push to the guest stack.
6688 */
6689DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6690{
6691 /*
6692 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6693 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6694 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6695 */
6696 if (pMixedCtx->sp == 1)
6697 return VINF_EM_RESET;
6698 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6699 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6700 AssertRCReturn(rc, rc);
6701 return rc;
6702}
6703
6704
6705/**
6706 * Injects an event into the guest upon VM-entry by updating the relevant fields
6707 * in the VM-entry area in the VMCS.
6708 *
6709 * @returns VBox status code (informational error codes included).
6710 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6711 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6712 *
6713 * @param pVCpu Pointer to the VMCPU.
6714 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6715 * be out-of-sync. Make sure to update the required
6716 * fields before using them.
6717 * @param u64IntrInfo The VM-entry interruption-information field.
6718 * @param cbInstr The VM-entry instruction length in bytes (for
6719 * software interrupts, exceptions and privileged
6720 * software exceptions).
6721 * @param u32ErrCode The VM-entry exception error code.
6722 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6723 * @param puIntrState Pointer to the current guest interruptibility-state.
6724 * This interruptibility-state will be updated if
6725 * necessary. This cannot not be NULL.
6726 *
6727 * @remarks Requires CR0!
6728 * @remarks No-long-jump zone!!!
6729 */
6730static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6731 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6732{
6733 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6734 AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6735 Assert(puIntrState);
6736 uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6737
6738 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6739 const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6740
6741#ifdef VBOX_STRICT
6742 /* Validate the error-code-valid bit for hardware exceptions. */
6743 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6744 {
6745 switch (uVector)
6746 {
6747 case X86_XCPT_PF:
6748 case X86_XCPT_DF:
6749 case X86_XCPT_TS:
6750 case X86_XCPT_NP:
6751 case X86_XCPT_SS:
6752 case X86_XCPT_GP:
6753 case X86_XCPT_AC:
6754 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6755 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6756 /* fallthru */
6757 default:
6758 break;
6759 }
6760 }
6761#endif
6762
6763 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
6764 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6765 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6766
6767 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6768
6769 /* We require CR0 to check if the guest is in real-mode. */
6770 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6771 AssertRCReturn(rc, rc);
6772
6773 /*
6774 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6775 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6776 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6777 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6778 */
6779 if (CPUMIsGuestInRealModeEx(pMixedCtx))
6780 {
6781 PVM pVM = pVCpu->CTX_SUFF(pVM);
6782 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6783 {
6784 Assert(PDMVmmDevHeapIsEnabled(pVM));
6785 Assert(pVM->hm.s.vmx.pRealModeTSS);
6786
6787 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6788 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6789 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6790 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6791 AssertRCReturn(rc, rc);
6792 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6793
6794 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6795 const size_t cbIdtEntry = sizeof(X86IDTR16);
6796 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6797 {
6798 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6799 if (uVector == X86_XCPT_DF)
6800 return VINF_EM_RESET;
6801 else if (uVector == X86_XCPT_GP)
6802 {
6803 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6804 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6805 }
6806
6807 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6808 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6809 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
6810 }
6811
6812 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6813 uint16_t uGuestIp = pMixedCtx->ip;
6814 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6815 {
6816 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
6817 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
6818 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6819 }
6820 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6821 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6822
6823 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6824 X86IDTR16 IdtEntry;
6825 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6826 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
6827 AssertRCReturn(rc, rc);
6828
6829 /* Construct the stack frame for the interrupt/exception handler. */
6830 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6831 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6832 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6833 AssertRCReturn(rc, rc);
6834
6835 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
6836 if (rc == VINF_SUCCESS)
6837 {
6838 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
6839 pMixedCtx->rip = IdtEntry.offSel;
6840 pMixedCtx->cs.Sel = IdtEntry.uSel;
6841 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
6842 if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6843 && uVector == X86_XCPT_PF)
6844 {
6845 pMixedCtx->cr2 = GCPtrFaultAddress;
6846 }
6847
6848 /* If any other guest-state bits are changed here, make sure to update
6849 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6850 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS
6851 | HM_CHANGED_GUEST_RIP
6852 | HM_CHANGED_GUEST_RFLAGS
6853 | HM_CHANGED_GUEST_RSP;
6854
6855 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6856 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6857 {
6858 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6859 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6860 Log4(("Clearing inhibition due to STI.\n"));
6861 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6862 }
6863 Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6864
6865 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
6866 it, if we are returning to ring-3 before executing guest code. */
6867 pVCpu->hm.s.Event.fPending = false;
6868 }
6869 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6870 return rc;
6871 }
6872 else
6873 {
6874 /*
6875 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6876 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6877 */
6878 u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6879 }
6880 }
6881
6882 /* Validate. */
6883 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6884 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6885 Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6886
6887 /* Inject. */
6888 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6889 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6890 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6891 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6892
6893 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6894 && uVector == X86_XCPT_PF)
6895 {
6896 pMixedCtx->cr2 = GCPtrFaultAddress;
6897 }
6898
6899 Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6900 u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6901
6902 AssertRCReturn(rc, rc);
6903 return rc;
6904}
6905
6906
6907/**
6908 * Clears the current event in the VMCS.
6909 *
6910 * @returns VBox status code.
6911 * @param pVCpu Pointer to the VMCPU.
6912 *
6913 * @remarks Use this function only to clear events that have not yet been
6914 * delivered to the guest but are injected in the VMCS!
6915 * @remarks No-long-jump zone!!!
6916 */
6917static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
6918{
6919 if (!pVCpu->hm.s.Event.fPending)
6920 return;
6921
6922#ifdef VBOX_STRICT
6923 uint32_t u32EntryInfo;
6924 int rc2 = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
6925 AssertRC(rc2);
6926 Assert(VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo));
6927#endif
6928
6929 /* Clear the entry-interruption field (including the valid bit). */
6930 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
6931 AssertRC(rc);
6932
6933 /* Clear the pending debug exception field. */
6934 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
6935 AssertRC(rc);
6936}
6937
6938
6939/**
6940 * Enters the VT-x session.
6941 *
6942 * @returns VBox status code.
6943 * @param pVM Pointer to the VM.
6944 * @param pVCpu Pointer to the VMCPU.
6945 * @param pCpu Pointer to the CPU info struct.
6946 */
6947VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
6948{
6949 AssertPtr(pVM);
6950 AssertPtr(pVCpu);
6951 Assert(pVM->hm.s.vmx.fSupported);
6952 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6953 NOREF(pCpu);
6954
6955 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6956 Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
6957
6958#ifdef VBOX_STRICT
6959 /* Make sure we're in VMX root mode. */
6960 RTCCUINTREG u32HostCR4 = ASMGetCR4();
6961 if (!(u32HostCR4 & X86_CR4_VMXE))
6962 {
6963 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6964 return VERR_VMX_X86_CR4_VMXE_CLEARED;
6965 }
6966#endif
6967
6968 /*
6969 * Load the VCPU's VMCS as the current (and active) one.
6970 */
6971 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
6972 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6973 if (RT_FAILURE(rc))
6974 return rc;
6975
6976 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
6977 pVCpu->hm.s.fLeaveDone = false;
6978 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
6979
6980 return VINF_SUCCESS;
6981}
6982
6983
6984/**
6985 * The thread-context callback (only on platforms which support it).
6986 *
6987 * @param enmEvent The thread-context event.
6988 * @param pVCpu Pointer to the VMCPU.
6989 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
6990 * @thread EMT.
6991 */
6992VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
6993{
6994 switch (enmEvent)
6995 {
6996 case RTTHREADCTXEVENT_PREEMPTING:
6997 {
6998 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6999 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7000 VMCPU_ASSERT_EMT(pVCpu);
7001
7002 PVM pVM = pVCpu->CTX_SUFF(pVM);
7003 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7004
7005 /* No longjmps (logger flushes, locks) in this fragile context. */
7006 VMMRZCallRing3Disable(pVCpu);
7007 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7008
7009 /* Save the guest-state, restore host-state (FPU, debug etc.). */
7010 if (!pVCpu->hm.s.fLeaveDone)
7011 {
7012 hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
7013 pVCpu->hm.s.fLeaveDone = true;
7014 }
7015
7016 /* Leave HM context, takes care of local init (term). */
7017 int rc = HMR0LeaveCpu(pVCpu);
7018 AssertRC(rc); NOREF(rc);
7019
7020 /* Restore longjmp state. */
7021 VMMRZCallRing3Enable(pVCpu);
7022 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7023 break;
7024 }
7025
7026 case RTTHREADCTXEVENT_RESUMED:
7027 {
7028 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7029 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7030 VMCPU_ASSERT_EMT(pVCpu);
7031
7032 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7033 VMMRZCallRing3Disable(pVCpu);
7034 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7035
7036 /* Initialize the bare minimum state required for HM. This takes care of
7037 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7038 int rc = HMR0EnterCpu(pVCpu);
7039 AssertRC(rc);
7040 Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7041
7042 /* Load the active VMCS as the current one. */
7043 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7044 {
7045 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7046 AssertRC(rc); NOREF(rc);
7047 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7048 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7049 }
7050 pVCpu->hm.s.fLeaveDone = false;
7051
7052 /* Restore longjmp state. */
7053 VMMRZCallRing3Enable(pVCpu);
7054 break;
7055 }
7056
7057 default:
7058 break;
7059 }
7060}
7061
7062
7063/**
7064 * Saves the host state in the VMCS host-state.
7065 * Sets up the VM-exit MSR-load area.
7066 *
7067 * The CPU state will be loaded from these fields on every successful VM-exit.
7068 *
7069 * @returns VBox status code.
7070 * @param pVM Pointer to the VM.
7071 * @param pVCpu Pointer to the VMCPU.
7072 *
7073 * @remarks No-long-jump zone!!!
7074 */
7075static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7076{
7077 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7078
7079 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
7080 return VINF_SUCCESS;
7081
7082 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7083 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7084
7085 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7086 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7087
7088 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7089 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7090
7091 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
7092 return rc;
7093}
7094
7095
7096/**
7097 * Saves the host state in the VMCS host-state.
7098 *
7099 * @returns VBox status code.
7100 * @param pVM Pointer to the VM.
7101 * @param pVCpu Pointer to the VMCPU.
7102 *
7103 * @remarks No-long-jump zone!!!
7104 */
7105VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7106{
7107 AssertPtr(pVM);
7108 AssertPtr(pVCpu);
7109
7110 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7111
7112 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7113 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7114 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7115 return hmR0VmxSaveHostState(pVM, pVCpu);
7116}
7117
7118
7119/**
7120 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7121 * loaded from these fields on every successful VM-entry.
7122 *
7123 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7124 * Sets up the VM-entry controls.
7125 * Sets up the appropriate VMX non-root function to execute guest code based on
7126 * the guest CPU mode.
7127 *
7128 * @returns VBox status code.
7129 * @param pVM Pointer to the VM.
7130 * @param pVCpu Pointer to the VMCPU.
7131 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7132 * out-of-sync. Make sure to update the required fields
7133 * before using them.
7134 *
7135 * @remarks No-long-jump zone!!!
7136 */
7137static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7138{
7139 AssertPtr(pVM);
7140 AssertPtr(pVCpu);
7141 AssertPtr(pMixedCtx);
7142 HMVMX_ASSERT_PREEMPT_SAFE();
7143
7144#ifdef LOG_ENABLED
7145 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7146 * probably not initialized yet? Anyway this will do for now.
7147 *
7148 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7149 * interface and disable ring-3 calls when thread-context hooks are not
7150 * available. */
7151 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7152 VMMR0LogFlushDisable(pVCpu);
7153#endif
7154
7155 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7156
7157 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7158
7159 /* Determine real-on-v86 mode. */
7160 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7161 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7162 && CPUMIsGuestInRealModeEx(pMixedCtx))
7163 {
7164 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7165 }
7166
7167 /*
7168 * Load the guest-state into the VMCS.
7169 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7170 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7171 */
7172 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7173 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7174
7175 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7176 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7177 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7178
7179 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
7180 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7181 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7182
7183 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7184 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7185
7186 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7187 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7188
7189 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7190 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7191 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7192
7193 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7194 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7195
7196 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7197 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7198
7199 /*
7200 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7201 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7202 */
7203 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7204 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7205
7206 /* Clear any unused and reserved bits. */
7207 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
7208
7209#ifdef LOG_ENABLED
7210 /* Only reenable log-flushing if the caller has it enabled. */
7211 if (!fCallerDisabledLogFlush)
7212 VMMR0LogFlushEnable(pVCpu);
7213#endif
7214
7215 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7216 return rc;
7217}
7218
7219
7220/**
7221 * Loads the state shared between the host and guest into the VMCS.
7222 *
7223 * @param pVM Pointer to the VM.
7224 * @param pVCpu Pointer to the VMCPU.
7225 * @param pCtx Pointer to the guest-CPU context.
7226 *
7227 * @remarks No-long-jump zone!!!
7228 */
7229static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7230{
7231 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7232 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7233
7234 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
7235 {
7236 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7237 AssertRC(rc);
7238 }
7239
7240 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
7241 {
7242 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7243 AssertRC(rc);
7244
7245 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7246 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
7247 {
7248 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7249 AssertRC(rc);
7250 }
7251 }
7252
7253 AssertMsg(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE), ("fContextUseFlags=%#x\n",
7254 pVCpu->hm.s.fContextUseFlags));
7255}
7256
7257
7258/**
7259 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7260 *
7261 * @param pVM Pointer to the VM.
7262 * @param pVCpu Pointer to the VMCPU.
7263 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7264 * out-of-sync. Make sure to update the required fields
7265 * before using them.
7266 */
7267DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7268{
7269 HMVMX_ASSERT_PREEMPT_SAFE();
7270
7271 Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7272#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7273 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
7274#endif
7275
7276 if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7277 {
7278 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7279 AssertRC(rc);
7280 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7281 }
7282 else if (pVCpu->hm.s.fContextUseFlags)
7283 {
7284 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7285 AssertRC(rc);
7286 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7287 }
7288
7289 /* All the guest state bits should be loaded except maybe the host context and shared host/guest bits. */
7290 AssertMsg( !(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST)
7291 || !(pVCpu->hm.s.fContextUseFlags & ~(HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE)),
7292 ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7293
7294#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7295 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7296 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7297 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7298#endif
7299}
7300
7301
7302/**
7303 * Does the preparations before executing guest code in VT-x.
7304 *
7305 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7306 * recompiler. We must be cautious what we do here regarding committing
7307 * guest-state information into the VMCS assuming we assuredly execute the
7308 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7309 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7310 * so that the recompiler can (and should) use them when it resumes guest
7311 * execution. Otherwise such operations must be done when we can no longer
7312 * exit to ring-3.
7313 *
7314 * @returns Strict VBox status code.
7315 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7316 * have been disabled.
7317 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7318 * double-fault into the guest.
7319 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7320 *
7321 * @param pVM Pointer to the VM.
7322 * @param pVCpu Pointer to the VMCPU.
7323 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7324 * out-of-sync. Make sure to update the required fields
7325 * before using them.
7326 * @param pVmxTransient Pointer to the VMX transient structure.
7327 *
7328 * @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7329 * interrupts will be disabled.
7330 */
7331static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7332{
7333 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7334
7335#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7336 PGMRZDynMapFlushAutoSet(pVCpu);
7337#endif
7338
7339 /* Check force flag actions that might require us to go back to ring-3. */
7340 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7341 if (rc != VINF_SUCCESS)
7342 return rc;
7343
7344#ifndef IEM_VERIFICATION_MODE_FULL
7345 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7346 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7347 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7348 {
7349 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7350 RTGCPHYS GCPhysApicBase;
7351 GCPhysApicBase = pMixedCtx->msrApicBase;
7352 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7353
7354 /* Unalias any existing mapping. */
7355 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7356 AssertRCReturn(rc, rc);
7357
7358 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7359 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7360 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7361 AssertRCReturn(rc, rc);
7362
7363 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7364 }
7365#endif /* !IEM_VERIFICATION_MODE_FULL */
7366
7367 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7368 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7369
7370 /*
7371 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7372 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7373 */
7374 if (TRPMHasTrap(pVCpu))
7375 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7376 else if (!pVCpu->hm.s.Event.fPending)
7377 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7378
7379 /*
7380 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
7381 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
7382 */
7383 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7384 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7385 {
7386 Assert(rc == VINF_EM_RESET);
7387 return rc;
7388 }
7389
7390 /*
7391 * No longjmps to ring-3 from this point on!!!
7392 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7393 * This also disables flushing of the R0-logger instance (if any).
7394 */
7395 VMMRZCallRing3Disable(pVCpu);
7396
7397 /*
7398 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7399 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7400 *
7401 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7402 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7403 *
7404 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7405 * executing guest code.
7406 */
7407 pVmxTransient->uEflags = ASMIntDisableFlags();
7408 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7409 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7410 {
7411 hmR0VmxClearEventVmcs(pVCpu);
7412 ASMSetFlags(pVmxTransient->uEflags);
7413 VMMRZCallRing3Enable(pVCpu);
7414 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7415 return VINF_EM_RAW_TO_R3;
7416 }
7417 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7418 {
7419 hmR0VmxClearEventVmcs(pVCpu);
7420 ASMSetFlags(pVmxTransient->uEflags);
7421 VMMRZCallRing3Enable(pVCpu);
7422 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7423 return VINF_EM_RAW_INTERRUPT;
7424 }
7425
7426 /* We've injected any pending events. This is really the point of no return (to ring-3). */
7427 pVCpu->hm.s.Event.fPending = false;
7428
7429 return VINF_SUCCESS;
7430}
7431
7432
7433/**
7434 * Prepares to run guest code in VT-x and we've committed to doing so. This
7435 * means there is no backing out to ring-3 or anywhere else at this
7436 * point.
7437 *
7438 * @param pVM Pointer to the VM.
7439 * @param pVCpu Pointer to the VMCPU.
7440 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7441 * out-of-sync. Make sure to update the required fields
7442 * before using them.
7443 * @param pVmxTransient Pointer to the VMX transient structure.
7444 *
7445 * @remarks Called with preemption disabled.
7446 * @remarks No-long-jump zone!!!
7447 */
7448static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7449{
7450 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7451 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7452 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7453
7454 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7455 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7456
7457 /*
7458 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7459 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7460 * Reload only the necessary state, the assertion will catch if other parts of the code
7461 * change.
7462 */
7463 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7464 {
7465 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7466 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7467 }
7468
7469 /*
7470 * Load the host state bits as we may've been preempted (only happens when
7471 * thread-context hooks are used).
7472 */
7473 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
7474 {
7475 /* This ASSUMES that pfnStartVM has been set up already. */
7476 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7477 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7478 AssertRC(rc);
7479 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
7480 }
7481 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7482
7483 /*
7484 * Load the state shared between host and guest (FPU, debug).
7485 */
7486 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE)
7487 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7488 AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7489
7490 /*
7491 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7492 */
7493 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7494 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7495
7496 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7497 RTCPUID idCurrentCpu = pCpu->idCpu;
7498 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7499 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
7500 {
7501 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7502 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7503 }
7504
7505 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7506 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7507 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7508 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7509
7510 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7511
7512 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7513 to start executing. */
7514
7515#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7516 /*
7517 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7518 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7519 */
7520 if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7521 && !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7522 {
7523 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7524 uint64_t u64HostTscAux = 0;
7525 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7526 AssertRC(rc2);
7527 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7528 }
7529#endif
7530}
7531
7532
7533/**
7534 * Performs some essential restoration of state after running guest code in
7535 * VT-x.
7536 *
7537 * @param pVM Pointer to the VM.
7538 * @param pVCpu Pointer to the VMCPU.
7539 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7540 * out-of-sync. Make sure to update the required fields
7541 * before using them.
7542 * @param pVmxTransient Pointer to the VMX transient structure.
7543 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7544 *
7545 * @remarks Called with interrupts disabled, and returns with interrups enabled!
7546 *
7547 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7548 * unconditionally when it is safe to do so.
7549 */
7550static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7551{
7552 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7553
7554 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7555 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7556 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7557 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7558 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7559
7560 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7561 {
7562#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7563 /* Restore host's TSC_AUX. */
7564 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7565 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7566#endif
7567 /** @todo Find a way to fix hardcoding a guestimate. */
7568 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7569 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7570 }
7571
7572 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7573 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7574 Assert(!(ASMGetFlags() & X86_EFL_IF));
7575 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7576
7577 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7578 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7579 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7580 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7581
7582 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7583 uint32_t uExitReason;
7584 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7585 rc |= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7586 AssertRC(rc);
7587 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7588 pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7589
7590 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7591 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7592 {
7593 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7594 pVmxTransient->fVMEntryFailed));
7595 return;
7596 }
7597
7598 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7599 {
7600 /* Update the guest interruptibility-state from the VMCS. */
7601 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7602#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7603 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7604 AssertRC(rc);
7605#endif
7606 /*
7607 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7608 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7609 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7610 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7611 */
7612 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7613 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7614 {
7615 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7616 AssertRC(rc);
7617 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
7618 }
7619 }
7620}
7621
7622
7623
7624/**
7625 * Runs the guest code using VT-x the normal way.
7626 *
7627 * @returns VBox status code.
7628 * @param pVM Pointer to the VM.
7629 * @param pVCpu Pointer to the VMCPU.
7630 * @param pCtx Pointer to the guest-CPU context.
7631 *
7632 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
7633 * @remarks Called with preemption disabled.
7634 */
7635static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7636{
7637 VMXTRANSIENT VmxTransient;
7638 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7639 int rc = VERR_INTERNAL_ERROR_5;
7640 uint32_t cLoops = 0;
7641
7642 for (;; cLoops++)
7643 {
7644 Assert(!HMR0SuspendPending());
7645 HMVMX_ASSERT_CPU_SAFE();
7646
7647 /* Preparatory work for running guest code, this may force us to return
7648 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7649 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7650 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7651 if (rc != VINF_SUCCESS)
7652 break;
7653
7654 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7655 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7656 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7657
7658 /* Restore any residual host-state and save any bits shared between host
7659 and guest into the guest-CPU state. Re-enables interrupts! */
7660 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7661
7662 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7663 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7664 {
7665 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7666 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7667 return rc;
7668 }
7669
7670 /* Handle the VM-exit. */
7671 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7672 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7673 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7674 HMVMX_START_EXIT_DISPATCH_PROF();
7675#ifdef HMVMX_USE_FUNCTION_TABLE
7676 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7677#else
7678 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7679#endif
7680 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7681 if (rc != VINF_SUCCESS)
7682 break;
7683 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7684 {
7685 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7686 rc = VINF_EM_RAW_INTERRUPT;
7687 break;
7688 }
7689 }
7690
7691 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7692 return rc;
7693}
7694
7695
7696/**
7697 * Single steps guest code using VT-x.
7698 *
7699 * @returns VBox status code.
7700 * @param pVM Pointer to the VM.
7701 * @param pVCpu Pointer to the VMCPU.
7702 * @param pCtx Pointer to the guest-CPU context.
7703 *
7704 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7705 * @remarks Called with preemption disabled.
7706 */
7707static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7708{
7709 VMXTRANSIENT VmxTransient;
7710 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7711 int rc = VERR_INTERNAL_ERROR_5;
7712 uint32_t cLoops = 0;
7713 uint16_t uCsStart = pCtx->cs.Sel;
7714 uint64_t uRipStart = pCtx->rip;
7715
7716 for (;; cLoops++)
7717 {
7718 Assert(!HMR0SuspendPending());
7719 HMVMX_ASSERT_CPU_SAFE();
7720
7721 /* Preparatory work for running guest code, this may force us to return
7722 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7723 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7724 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7725 if (rc != VINF_SUCCESS)
7726 break;
7727
7728 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7729 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7730 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7731
7732 /* Restore any residual host-state and save any bits shared between host
7733 and guest into the guest-CPU state. Re-enables interrupts! */
7734 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7735
7736 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7737 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7738 {
7739 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7740 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7741 return rc;
7742 }
7743
7744 /* Handle the VM-exit. */
7745 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7746 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7747 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7748 HMVMX_START_EXIT_DISPATCH_PROF();
7749#ifdef HMVMX_USE_FUNCTION_TABLE
7750 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7751#else
7752 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7753#endif
7754 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7755 if (rc != VINF_SUCCESS)
7756 break;
7757 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7758 {
7759 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7760 rc = VINF_EM_RAW_INTERRUPT;
7761 break;
7762 }
7763
7764 /*
7765 * Did the RIP change, if so, consider it a single step.
7766 * Otherwise, make sure one of the TFs gets set.
7767 */
7768 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7769 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7770 AssertRCReturn(rc2, rc2);
7771 if ( pCtx->rip != uRipStart
7772 || pCtx->cs.Sel != uCsStart)
7773 {
7774 rc = VINF_EM_DBG_STEPPED;
7775 break;
7776 }
7777 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
7778 }
7779
7780 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7781 return rc;
7782}
7783
7784
7785/**
7786 * Runs the guest code using VT-x.
7787 *
7788 * @returns VBox status code.
7789 * @param pVM Pointer to the VM.
7790 * @param pVCpu Pointer to the VMCPU.
7791 * @param pCtx Pointer to the guest-CPU context.
7792 *
7793 * @remarks Called with preemption disabled.
7794 */
7795VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7796{
7797 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7798 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
7799 HMVMX_ASSERT_PREEMPT_SAFE();
7800
7801 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
7802
7803 int rc;
7804 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7805 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7806 else
7807 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7808
7809 if (rc == VERR_EM_INTERPRETER)
7810 rc = VINF_EM_RAW_EMULATE_INSTR;
7811 else if (rc == VINF_EM_RESET)
7812 rc = VINF_EM_TRIPLE_FAULT;
7813
7814 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7815 if (RT_FAILURE(rc2))
7816 {
7817 pVCpu->hm.s.u32HMError = rc;
7818 rc = rc2;
7819 }
7820 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
7821 return rc;
7822}
7823
7824
7825#ifndef HMVMX_USE_FUNCTION_TABLE
7826DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7827{
7828 int rc;
7829 switch (rcReason)
7830 {
7831 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7832 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7833 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7834 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7835 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7836 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7837 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7838 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7839 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7840 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7841 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7842 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7843 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7844 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7845 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7846 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7847 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7848 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7849 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7850 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7851 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7852 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7853 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7854 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7855 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7856 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7857 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7858 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7859 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7860 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7861 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7862 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7863 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7864
7865 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7866 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7867 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7868 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7869 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7870 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7871 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7872 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7873 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7874
7875 case VMX_EXIT_VMCALL:
7876 case VMX_EXIT_VMCLEAR:
7877 case VMX_EXIT_VMLAUNCH:
7878 case VMX_EXIT_VMPTRLD:
7879 case VMX_EXIT_VMPTRST:
7880 case VMX_EXIT_VMREAD:
7881 case VMX_EXIT_VMRESUME:
7882 case VMX_EXIT_VMWRITE:
7883 case VMX_EXIT_VMXOFF:
7884 case VMX_EXIT_VMXON:
7885 case VMX_EXIT_INVEPT:
7886 case VMX_EXIT_INVVPID:
7887 case VMX_EXIT_VMFUNC:
7888 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7889 break;
7890 default:
7891 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7892 break;
7893 }
7894 return rc;
7895}
7896#endif
7897
7898#ifdef DEBUG
7899/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7900# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7901 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7902
7903# define HMVMX_ASSERT_PREEMPT_CPUID() \
7904 do \
7905 { \
7906 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7907 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7908 } while (0)
7909
7910# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7911 do { \
7912 AssertPtr(pVCpu); \
7913 AssertPtr(pMixedCtx); \
7914 AssertPtr(pVmxTransient); \
7915 Assert(pVmxTransient->fVMEntryFailed == false); \
7916 Assert(ASMIntAreEnabled()); \
7917 HMVMX_ASSERT_PREEMPT_SAFE(); \
7918 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7919 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7920 HMVMX_ASSERT_PREEMPT_SAFE(); \
7921 if (VMMR0IsLogFlushDisabled(pVCpu)) \
7922 HMVMX_ASSERT_PREEMPT_CPUID(); \
7923 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7924 } while (0)
7925
7926# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7927 do { \
7928 Log4Func(("\n")); \
7929 } while(0)
7930#else /* Release builds */
7931# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7932# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7933#endif
7934
7935
7936/**
7937 * Advances the guest RIP after reading it from the VMCS.
7938 *
7939 * @returns VBox status code.
7940 * @param pVCpu Pointer to the VMCPU.
7941 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7942 * out-of-sync. Make sure to update the required fields
7943 * before using them.
7944 * @param pVmxTransient Pointer to the VMX transient structure.
7945 *
7946 * @remarks No-long-jump zone!!!
7947 */
7948DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7949{
7950 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7951 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7952 AssertRCReturn(rc, rc);
7953
7954 pMixedCtx->rip += pVmxTransient->cbInstr;
7955 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
7956 return rc;
7957}
7958
7959
7960/**
7961 * Tries to determine what part of the guest-state VT-x has deemed as invalid
7962 * and update error record fields accordingly.
7963 *
7964 * @return VMX_IGS_* return codes.
7965 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7966 * wrong with the guest state.
7967 *
7968 * @param pVM Pointer to the VM.
7969 * @param pVCpu Pointer to the VMCPU.
7970 * @param pCtx Pointer to the guest-CPU state.
7971 */
7972static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7973{
7974#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7975#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
7976 uError = (err); \
7977 break; \
7978 } else do {} while (0)
7979/* Duplicate of IEM_IS_CANONICAL(). */
7980#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
7981
7982 int rc;
7983 uint64_t u64Val;
7984 uint32_t u32Val;
7985 uint32_t uError = VMX_IGS_ERROR;
7986 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
7987
7988 do
7989 {
7990 /*
7991 * CR0.
7992 */
7993 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
7994 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
7995 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
7996 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
7997 if (fUnrestrictedGuest)
7998 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
7999
8000 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
8001 AssertRCBreak(rc);
8002 HMVMX_CHECK_BREAK((u32Val & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8003 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8004 if ( !fUnrestrictedGuest
8005 && (u32Val & X86_CR0_PG)
8006 && !(u32Val & X86_CR0_PE))
8007 {
8008 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8009 }
8010
8011 /*
8012 * CR4.
8013 */
8014 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8015 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
8016 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
8017 AssertRCBreak(rc);
8018 HMVMX_CHECK_BREAK((u32Val & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
8019 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR4), VMX_IGS_CR4_FIXED0);
8020
8021 /*
8022 * IA32_DEBUGCTL MSR.
8023 */
8024 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
8025 AssertRCBreak(rc);
8026 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8027 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
8028 {
8029 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
8030 }
8031 uint64_t u64DebugCtlMsr = u64Val;
8032
8033#ifdef VBOX_STRICT
8034 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
8035 AssertRCBreak(rc);
8036 Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8037#endif
8038 bool const fLongModeGuest = !!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8039
8040 /*
8041 * RIP and RFLAGS.
8042 */
8043 uint32_t u32Eflags;
8044#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8045 if (HMVMX_IS_64BIT_HOST_MODE())
8046 {
8047 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8048 AssertRCBreak(rc);
8049 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8050 if ( !fLongModeGuest
8051 || !pCtx->cs.Attr.n.u1Long)
8052 {
8053 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8054 }
8055 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8056 * must be identical if the "IA32e mode guest" VM-entry control is 1
8057 * and CS.L is 1. No check applies if the CPU supports 64
8058 * linear-address bits. */
8059
8060 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8061 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8062 AssertRCBreak(rc);
8063 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8064 VMX_IGS_RFLAGS_RESERVED);
8065 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8066 u32Eflags = u64Val;
8067 }
8068 else
8069#endif
8070 {
8071 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8072 AssertRCBreak(rc);
8073 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8074 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8075 }
8076
8077 if ( fLongModeGuest
8078 || !(pCtx->cr0 & X86_CR0_PE))
8079 {
8080 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8081 }
8082
8083 uint32_t u32EntryInfo;
8084 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8085 AssertRCBreak(rc);
8086 if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8087 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8088 {
8089 HMVMX_CHECK_BREAK(u32Val & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8090 }
8091
8092 /*
8093 * 64-bit checks.
8094 */
8095#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8096 if (HMVMX_IS_64BIT_HOST_MODE())
8097 {
8098 if ( fLongModeGuest
8099 && !fUnrestrictedGuest)
8100 {
8101 HMVMX_CHECK_BREAK(CPUMIsGuestPagingEnabledEx(pCtx), VMX_IGS_CR0_PG_LONGMODE);
8102 HMVMX_CHECK_BREAK((pCtx->cr4 & X86_CR4_PAE), VMX_IGS_CR4_PAE_LONGMODE);
8103 }
8104
8105 if ( !fLongModeGuest
8106 && (pCtx->cr4 & X86_CR4_PCIDE))
8107 {
8108 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8109 }
8110
8111 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8112 * 51:32 beyond the processor's physical-address width are 0. */
8113
8114 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8115 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8116 {
8117 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8118 }
8119
8120 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8121 AssertRCBreak(rc);
8122 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8123
8124 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8125 AssertRCBreak(rc);
8126 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8127 }
8128#endif
8129
8130 /*
8131 * PERF_GLOBAL MSR.
8132 */
8133 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8134 {
8135 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8136 AssertRCBreak(rc);
8137 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8138 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8139 }
8140
8141 /*
8142 * PAT MSR.
8143 */
8144 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8145 {
8146 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8147 AssertRCBreak(rc);
8148 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8149 for (unsigned i = 0; i < 8; i++)
8150 {
8151 uint8_t u8Val = (u64Val & 0x7);
8152 if ( u8Val != 0 /* UC */
8153 || u8Val != 1 /* WC */
8154 || u8Val != 4 /* WT */
8155 || u8Val != 5 /* WP */
8156 || u8Val != 6 /* WB */
8157 || u8Val != 7 /* UC- */)
8158 {
8159 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8160 }
8161 u64Val >>= 3;
8162 }
8163 }
8164
8165 /*
8166 * EFER MSR.
8167 */
8168 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8169 {
8170 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8171 AssertRCBreak(rc);
8172 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8173 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8174 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8175 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8176 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8177 || (u64Val & MSR_K6_EFER_LMA) == (pCtx->cr0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8178 }
8179
8180 /*
8181 * Segment registers.
8182 */
8183 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8184 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8185 if (!(u32Eflags & X86_EFL_VM))
8186 {
8187 /* CS */
8188 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8189 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8190 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8191 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8192 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8193 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8194 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8195 /* CS cannot be loaded with NULL in protected mode. */
8196 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8197 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8198 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8199 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8200 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8201 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8202 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8203 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8204 else
8205 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8206
8207 /* SS */
8208 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8209 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8210 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8211 if ( !(pCtx->cr0 & X86_CR0_PE)
8212 || pCtx->cs.Attr.n.u4Type == 3)
8213 {
8214 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8215 }
8216 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8217 {
8218 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8219 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8220 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8221 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8222 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8223 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8224 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8225 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8226 }
8227
8228 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8229 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8230 {
8231 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8232 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8233 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8234 || pCtx->ds.Attr.n.u4Type > 11
8235 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8236 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8237 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8238 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8239 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8240 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8241 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8242 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8243 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8244 }
8245 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8246 {
8247 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8248 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8249 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8250 || pCtx->es.Attr.n.u4Type > 11
8251 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8252 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8253 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8254 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8255 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8256 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8257 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8258 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8259 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8260 }
8261 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8262 {
8263 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8264 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8265 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8266 || pCtx->fs.Attr.n.u4Type > 11
8267 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8268 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8269 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8270 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8271 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8272 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8273 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8274 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8275 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8276 }
8277 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8278 {
8279 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8280 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8281 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8282 || pCtx->gs.Attr.n.u4Type > 11
8283 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8284 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8285 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8286 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8287 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8288 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8289 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8290 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8291 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8292 }
8293 /* 64-bit capable CPUs. */
8294#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8295 if (HMVMX_IS_64BIT_HOST_MODE())
8296 {
8297 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8298 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8299 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8300 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8301 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8302 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8303 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8304 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8305 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8306 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8307 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8308 }
8309#endif
8310 }
8311 else
8312 {
8313 /* V86 mode checks. */
8314 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8315 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8316 {
8317 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8318 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8319 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8320 }
8321 else
8322 {
8323 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8324 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8325 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8326 }
8327
8328 /* CS */
8329 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8330 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8331 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8332 /* SS */
8333 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8334 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8335 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8336 /* DS */
8337 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8338 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8339 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8340 /* ES */
8341 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8342 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8343 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8344 /* FS */
8345 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8346 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8347 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8348 /* GS */
8349 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8350 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8351 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8352 /* 64-bit capable CPUs. */
8353#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8354 if (HMVMX_IS_64BIT_HOST_MODE())
8355 {
8356 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8357 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8358 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8359 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8360 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8361 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8362 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8363 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8364 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8365 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8366 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8367 }
8368#endif
8369 }
8370
8371 /*
8372 * TR.
8373 */
8374 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8375 /* 64-bit capable CPUs. */
8376#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8377 if (HMVMX_IS_64BIT_HOST_MODE())
8378 {
8379 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8380 }
8381#endif
8382 if (fLongModeGuest)
8383 {
8384 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8385 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8386 }
8387 else
8388 {
8389 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8390 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8391 VMX_IGS_TR_ATTR_TYPE_INVALID);
8392 }
8393 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8394 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8395 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8396 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8397 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8398 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8399 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8400 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8401
8402 /*
8403 * GDTR and IDTR.
8404 */
8405#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8406 if (HMVMX_IS_64BIT_HOST_MODE())
8407 {
8408 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8409 AssertRCBreak(rc);
8410 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8411
8412 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8413 AssertRCBreak(rc);
8414 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8415 }
8416#endif
8417
8418 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8419 AssertRCBreak(rc);
8420 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8421
8422 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8423 AssertRCBreak(rc);
8424 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8425
8426 /*
8427 * Guest Non-Register State.
8428 */
8429 /* Activity State. */
8430 uint32_t u32ActivityState;
8431 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8432 AssertRCBreak(rc);
8433 HMVMX_CHECK_BREAK( !u32ActivityState
8434 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8435 VMX_IGS_ACTIVITY_STATE_INVALID);
8436 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8437 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8438 uint32_t u32IntrState;
8439 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8440 AssertRCBreak(rc);
8441 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8442 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8443 {
8444 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8445 }
8446
8447 /** @todo Activity state and injecting interrupts. Left as a todo since we
8448 * currently don't use activity states but ACTIVE. */
8449
8450 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8451 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8452
8453 /* Guest interruptibility-state. */
8454 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8455 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8456 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8457 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8458 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8459 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8460 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8461 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8462 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8463 if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8464 {
8465 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8466 {
8467 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8468 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8469 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8470 }
8471 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8472 {
8473 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8474 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8475 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8476 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8477 }
8478 }
8479 /** @todo Assumes the processor is not in SMM. */
8480 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8481 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8482 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8483 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8484 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8485 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8486 && VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8487 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8488 {
8489 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8490 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8491 }
8492
8493 /* Pending debug exceptions. */
8494 if (HMVMX_IS_64BIT_HOST_MODE())
8495 {
8496 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8497 AssertRCBreak(rc);
8498 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8499 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8500 u32Val = u64Val; /* For pending debug exceptions checks below. */
8501 }
8502 else
8503 {
8504 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8505 AssertRCBreak(rc);
8506 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8507 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8508 }
8509
8510 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8511 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8512 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8513 {
8514 if ( (u32Eflags & X86_EFL_TF)
8515 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8516 {
8517 /* Bit 14 is PendingDebug.BS. */
8518 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8519 }
8520 if ( !(u32Eflags & X86_EFL_TF)
8521 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8522 {
8523 /* Bit 14 is PendingDebug.BS. */
8524 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8525 }
8526 }
8527
8528 /* VMCS link pointer. */
8529 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8530 AssertRCBreak(rc);
8531 if (u64Val != UINT64_C(0xffffffffffffffff))
8532 {
8533 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8534 /** @todo Bits beyond the processor's physical-address width MBZ. */
8535 /** @todo 32-bit located in memory referenced by value of this field (as a
8536 * physical address) must contain the processor's VMCS revision ID. */
8537 /** @todo SMM checks. */
8538 }
8539
8540 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8541
8542 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8543 if (uError == VMX_IGS_ERROR)
8544 uError = VMX_IGS_REASON_NOT_FOUND;
8545 } while (0);
8546
8547 pVCpu->hm.s.u32HMError = uError;
8548 return uError;
8549
8550#undef HMVMX_ERROR_BREAK
8551#undef HMVMX_CHECK_BREAK
8552#undef HMVMX_IS_CANONICAL
8553}
8554
8555/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8556/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8557/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8558
8559/** @name VM-exit handlers.
8560 * @{
8561 */
8562
8563/**
8564 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8565 */
8566HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8567{
8568 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8569 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8570 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8571#if HC_ARCH_BITS == 64
8572 Assert(ASMIntAreEnabled());
8573 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8574 return VINF_SUCCESS;
8575#endif
8576 return VINF_EM_RAW_INTERRUPT;
8577}
8578
8579
8580/**
8581 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8582 */
8583HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8584{
8585 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8586 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8587
8588 int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8589 AssertRCReturn(rc, rc);
8590
8591 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8592 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8593 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8594 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8595
8596 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8597 {
8598 /*
8599 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8600 * anything we inject is not going to cause a VM-exit directly for the event being injected.
8601 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8602 *
8603 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8604 */
8605 VMXDispatchHostNmi();
8606 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
8607 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8608 return VINF_SUCCESS;
8609 }
8610
8611 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8612 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8613 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8614 {
8615 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8616 return VINF_SUCCESS;
8617 }
8618 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8619 {
8620 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8621 return rc;
8622 }
8623
8624 uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8625 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8626 switch (uIntrType)
8627 {
8628 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8629 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8630 /* no break */
8631 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8632 {
8633 switch (uVector)
8634 {
8635 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8636 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8637 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8638 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8639 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8640 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8641#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8642 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8643 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8644 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8645 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8646 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8647 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8648 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8649 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8650 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8651 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8652#endif
8653 default:
8654 {
8655 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8656 AssertRCReturn(rc, rc);
8657
8658 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8659 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8660 {
8661 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8662 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8663 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8664
8665 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8666 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8667 AssertRCReturn(rc, rc);
8668 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8669 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8670 0 /* GCPtrFaultAddress */);
8671 AssertRCReturn(rc, rc);
8672 }
8673 else
8674 {
8675 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8676 pVCpu->hm.s.u32HMError = uVector;
8677 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8678 }
8679 break;
8680 }
8681 }
8682 break;
8683 }
8684
8685 default:
8686 {
8687 pVCpu->hm.s.u32HMError = uExitIntrInfo;
8688 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8689 AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8690 break;
8691 }
8692 }
8693 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8694 return rc;
8695}
8696
8697
8698/**
8699 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8700 */
8701HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8702{
8703 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8704
8705 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8706 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8707 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8708 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8709 AssertRCReturn(rc, rc);
8710
8711 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8712 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8713 return VINF_SUCCESS;
8714}
8715
8716
8717/**
8718 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8719 */
8720HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8721{
8722 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8723 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8724 pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8725 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8726}
8727
8728
8729/**
8730 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8731 */
8732HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8733{
8734 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8735 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8736 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8737}
8738
8739
8740/**
8741 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8742 */
8743HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8744{
8745 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8746 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8747 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8748}
8749
8750
8751/**
8752 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8753 */
8754HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8755{
8756 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8757 PVM pVM = pVCpu->CTX_SUFF(pVM);
8758 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8759 if (RT_LIKELY(rc == VINF_SUCCESS))
8760 {
8761 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8762 Assert(pVmxTransient->cbInstr == 2);
8763 }
8764 else
8765 {
8766 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8767 rc = VERR_EM_INTERPRETER;
8768 }
8769 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8770 return rc;
8771}
8772
8773
8774/**
8775 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8776 */
8777HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8778{
8779 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8780 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8781 AssertRCReturn(rc, rc);
8782
8783 if (pMixedCtx->cr4 & X86_CR4_SMXE)
8784 return VINF_EM_RAW_EMULATE_INSTR;
8785
8786 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8787 pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8788 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8789}
8790
8791
8792/**
8793 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8794 */
8795HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8796{
8797 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8798 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8799 AssertRCReturn(rc, rc);
8800
8801 PVM pVM = pVCpu->CTX_SUFF(pVM);
8802 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8803 if (RT_LIKELY(rc == VINF_SUCCESS))
8804 {
8805 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8806 Assert(pVmxTransient->cbInstr == 2);
8807 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8808 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8809 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8810 }
8811 else
8812 {
8813 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8814 rc = VERR_EM_INTERPRETER;
8815 }
8816 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8817 return rc;
8818}
8819
8820
8821/**
8822 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8823 */
8824HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8825{
8826 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8827 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8828 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8829 AssertRCReturn(rc, rc);
8830
8831 PVM pVM = pVCpu->CTX_SUFF(pVM);
8832 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8833 if (RT_LIKELY(rc == VINF_SUCCESS))
8834 {
8835 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8836 Assert(pVmxTransient->cbInstr == 3);
8837 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8838 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8839 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8840 }
8841 else
8842 {
8843 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8844 rc = VERR_EM_INTERPRETER;
8845 }
8846 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8847 return rc;
8848}
8849
8850
8851/**
8852 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8853 */
8854HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8855{
8856 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8857 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8858 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8859 AssertRCReturn(rc, rc);
8860
8861 PVM pVM = pVCpu->CTX_SUFF(pVM);
8862 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8863 if (RT_LIKELY(rc == VINF_SUCCESS))
8864 {
8865 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8866 Assert(pVmxTransient->cbInstr == 2);
8867 }
8868 else
8869 {
8870 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8871 rc = VERR_EM_INTERPRETER;
8872 }
8873 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8874 return rc;
8875}
8876
8877
8878/**
8879 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8880 */
8881HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8882{
8883 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8884 PVM pVM = pVCpu->CTX_SUFF(pVM);
8885 Assert(!pVM->hm.s.fNestedPaging);
8886
8887 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8888 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8889 AssertRCReturn(rc, rc);
8890
8891 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8892 rc = VBOXSTRICTRC_VAL(rc2);
8893 if (RT_LIKELY(rc == VINF_SUCCESS))
8894 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8895 else
8896 {
8897 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8898 pVmxTransient->uExitQualification, rc));
8899 }
8900 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8901 return rc;
8902}
8903
8904
8905/**
8906 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8907 */
8908HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8909{
8910 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8911 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8912 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8913 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8914 AssertRCReturn(rc, rc);
8915
8916 PVM pVM = pVCpu->CTX_SUFF(pVM);
8917 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8918 if (RT_LIKELY(rc == VINF_SUCCESS))
8919 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8920 else
8921 {
8922 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8923 rc = VERR_EM_INTERPRETER;
8924 }
8925 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8926 return rc;
8927}
8928
8929
8930/**
8931 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8932 */
8933HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8934{
8935 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8936 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8937 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8938 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8939 AssertRCReturn(rc, rc);
8940
8941 PVM pVM = pVCpu->CTX_SUFF(pVM);
8942 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8943 rc = VBOXSTRICTRC_VAL(rc2);
8944 if (RT_LIKELY( rc == VINF_SUCCESS
8945 || rc == VINF_EM_HALT))
8946 {
8947 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8948 AssertRCReturn(rc3, rc3);
8949
8950 if ( rc == VINF_EM_HALT
8951 && EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
8952 {
8953 rc = VINF_SUCCESS;
8954 }
8955 }
8956 else
8957 {
8958 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
8959 rc = VERR_EM_INTERPRETER;
8960 }
8961 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
8962 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
8963 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
8964 return rc;
8965}
8966
8967
8968/**
8969 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
8970 */
8971HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8972{
8973 /*
8974 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
8975 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
8976 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
8977 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
8978 */
8979 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8980 pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
8981 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8982}
8983
8984
8985/**
8986 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
8987 */
8988HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8989{
8990 /*
8991 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
8992 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
8993 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
8994 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
8995 */
8996 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8997 pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
8998 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8999}
9000
9001
9002/**
9003 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
9004 */
9005HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9006{
9007 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
9008 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9009 pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
9010 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9011}
9012
9013
9014/**
9015 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
9016 */
9017HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9018{
9019 /*
9020 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
9021 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
9022 * See Intel spec. 25.3 "Other Causes of VM-exits".
9023 */
9024 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9025 pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
9026 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9027}
9028
9029
9030/**
9031 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
9032 * VM-exit.
9033 */
9034HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9035{
9036 /*
9037 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9038 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9039 *
9040 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9041 * See Intel spec. "23.8 Restrictions on VMX operation".
9042 */
9043 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9044 return VINF_SUCCESS;
9045}
9046
9047
9048/**
9049 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9050 * VM-exit.
9051 */
9052HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9053{
9054 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9055 return VINF_EM_RESET;
9056}
9057
9058
9059/**
9060 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9061 */
9062HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9063{
9064 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9065 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9066 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9067 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9068 AssertRCReturn(rc, rc);
9069
9070 pMixedCtx->rip++;
9071 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9072 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9073 rc = VINF_SUCCESS;
9074 else
9075 rc = VINF_EM_HALT;
9076
9077 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9078 return rc;
9079}
9080
9081
9082/**
9083 * VM-exit handler for instructions that result in a #UD exception delivered to
9084 * the guest.
9085 */
9086HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9087{
9088 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9089 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9090 return VINF_SUCCESS;
9091}
9092
9093
9094/**
9095 * VM-exit handler for expiry of the VMX preemption timer.
9096 */
9097HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9098{
9099 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9100
9101 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9102 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9103
9104 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9105 PVM pVM = pVCpu->CTX_SUFF(pVM);
9106 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9107 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9108 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9109}
9110
9111
9112/**
9113 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9114 */
9115HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9116{
9117 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9118
9119 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9120 /** @todo check if XSETBV is supported by the recompiler. */
9121 return VERR_EM_INTERPRETER;
9122}
9123
9124
9125/**
9126 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9127 */
9128HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9129{
9130 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9131
9132 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9133 /** @todo implement EMInterpretInvpcid() */
9134 return VERR_EM_INTERPRETER;
9135}
9136
9137
9138/**
9139 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9140 * Error VM-exit.
9141 */
9142HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9143{
9144 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9145 AssertRCReturn(rc, rc);
9146
9147 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9148 NOREF(uInvalidReason);
9149
9150#ifdef VBOX_STRICT
9151 uint32_t uIntrState;
9152 HMVMXHCUINTREG uHCReg;
9153 uint64_t u64Val;
9154 uint32_t u32Val;
9155
9156 rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
9157 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9158 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9159 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9160 AssertRCReturn(rc, rc);
9161
9162 Log4(("uInvalidReason %u\n", uInvalidReason));
9163 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
9164 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9165 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9166 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9167
9168 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9169 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9170 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9171 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9172 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9173 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9174 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9175 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9176 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9177 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9178 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9179 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9180#endif
9181
9182 PVM pVM = pVCpu->CTX_SUFF(pVM);
9183 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9184
9185 return VERR_VMX_INVALID_GUEST_STATE;
9186}
9187
9188
9189/**
9190 * VM-exit handler for VM-entry failure due to an MSR-load
9191 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9192 */
9193HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9194{
9195 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9196 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9197}
9198
9199
9200/**
9201 * VM-exit handler for VM-entry failure due to a machine-check event
9202 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9203 */
9204HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9205{
9206 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9207 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9208}
9209
9210
9211/**
9212 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9213 * theory.
9214 */
9215HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9216{
9217 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9218 return VERR_VMX_UNDEFINED_EXIT_CODE;
9219}
9220
9221
9222/**
9223 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9224 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9225 * Conditional VM-exit.
9226 */
9227HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9228{
9229 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9230
9231 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9232 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9233 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9234 return VERR_EM_INTERPRETER;
9235 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9236 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9237}
9238
9239
9240/**
9241 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9242 */
9243HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9244{
9245 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9246
9247 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9248 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9249 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9250 return VERR_EM_INTERPRETER;
9251 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9252 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9253}
9254
9255
9256/**
9257 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9258 */
9259HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9260{
9261 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9262
9263 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9264 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9265 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9266 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9267 AssertRCReturn(rc, rc);
9268 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9269
9270 PVM pVM = pVCpu->CTX_SUFF(pVM);
9271 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9272 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9273 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9274 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9275
9276 if (RT_LIKELY(rc == VINF_SUCCESS))
9277 {
9278 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9279 Assert(pVmxTransient->cbInstr == 2);
9280 }
9281 return rc;
9282}
9283
9284
9285/**
9286 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9287 */
9288HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9289{
9290 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9291 PVM pVM = pVCpu->CTX_SUFF(pVM);
9292 int rc = VINF_SUCCESS;
9293
9294 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9295 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9296 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9297 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9298 AssertRCReturn(rc, rc);
9299 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9300
9301 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9302 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9303 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9304
9305 if (RT_LIKELY(rc == VINF_SUCCESS))
9306 {
9307 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9308
9309 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9310 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9311 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9312 {
9313 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9314 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9315 EMInterpretWrmsr() changes it. */
9316 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9317 }
9318 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9319 {
9320 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9321 AssertRCReturn(rc, rc);
9322 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
9323 }
9324 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9325 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9326
9327 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9328 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9329 {
9330 switch (pMixedCtx->ecx)
9331 {
9332 case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
9333 case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
9334 case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
9335 case MSR_K8_FS_BASE: /* no break */
9336 case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS; break;
9337 case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
9338 }
9339 }
9340#ifdef VBOX_STRICT
9341 else
9342 {
9343 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9344 switch (pMixedCtx->ecx)
9345 {
9346 case MSR_IA32_SYSENTER_CS:
9347 case MSR_IA32_SYSENTER_EIP:
9348 case MSR_IA32_SYSENTER_ESP:
9349 case MSR_K8_FS_BASE:
9350 case MSR_K8_GS_BASE:
9351 {
9352 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9353 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9354 }
9355
9356 case MSR_K8_LSTAR:
9357 case MSR_K6_STAR:
9358 case MSR_K8_SF_MASK:
9359 case MSR_K8_TSC_AUX:
9360 case MSR_K8_KERNEL_GS_BASE:
9361 {
9362 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9363 pMixedCtx->ecx));
9364 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9365 }
9366 }
9367 }
9368#endif /* VBOX_STRICT */
9369 }
9370 return rc;
9371}
9372
9373
9374/**
9375 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9376 */
9377HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9378{
9379 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9380
9381 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9382 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9383 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9384 return VERR_EM_INTERPRETER;
9385 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9386 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9387}
9388
9389
9390/**
9391 * VM-exit handler for when the TPR value is lowered below the specified
9392 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9393 */
9394HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9395{
9396 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9397 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9398
9399 /*
9400 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9401 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9402 * resume guest execution.
9403 */
9404 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9405 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9406 return VINF_SUCCESS;
9407}
9408
9409
9410/**
9411 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9412 * VM-exit.
9413 *
9414 * @retval VINF_SUCCESS when guest execution can continue.
9415 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9416 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9417 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9418 * recompiler.
9419 */
9420HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9421{
9422 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9423 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9424 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9425 AssertRCReturn(rc, rc);
9426
9427 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9428 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9429 PVM pVM = pVCpu->CTX_SUFF(pVM);
9430 switch (uAccessType)
9431 {
9432 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9433 {
9434#if 0
9435 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9436 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9437#else
9438 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9439 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9440 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9441#endif
9442 AssertRCReturn(rc, rc);
9443
9444 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9445 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9446 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9447 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9448
9449 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9450 {
9451 case 0: /* CR0 */
9452 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9453 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9454 break;
9455 case 2: /* C2 **/
9456 /* Nothing to do here, CR2 it's not part of the VMCS. */
9457 break;
9458 case 3: /* CR3 */
9459 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9460 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9461 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR3;
9462 break;
9463 case 4: /* CR4 */
9464 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9465 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR4;
9466 break;
9467 case 8: /* CR8 */
9468 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9469 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9470 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9471 break;
9472 default:
9473 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9474 break;
9475 }
9476
9477 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9478 break;
9479 }
9480
9481 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9482 {
9483 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9484 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9485 AssertRCReturn(rc, rc);
9486 Assert( !pVM->hm.s.fNestedPaging
9487 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9488 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9489
9490 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9491 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9492 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9493
9494 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9495 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9496 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9497 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9498 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9499 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9500 break;
9501 }
9502
9503 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9504 {
9505 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9506 AssertRCReturn(rc, rc);
9507 rc = EMInterpretCLTS(pVM, pVCpu);
9508 AssertRCReturn(rc, rc);
9509 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9510 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9511 Log4(("CRX CLTS write rc=%d\n", rc));
9512 break;
9513 }
9514
9515 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9516 {
9517 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9518 AssertRCReturn(rc, rc);
9519 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9520 if (RT_LIKELY(rc == VINF_SUCCESS))
9521 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9522 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9523 Log4(("CRX LMSW write rc=%d\n", rc));
9524 break;
9525 }
9526
9527 default:
9528 {
9529 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9530 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9531 }
9532 }
9533
9534 /* Validate possible error codes. */
9535 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9536 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9537 if (RT_SUCCESS(rc))
9538 {
9539 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9540 AssertRCReturn(rc2, rc2);
9541 }
9542
9543 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9544 return rc;
9545}
9546
9547
9548/**
9549 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9550 * VM-exit.
9551 */
9552HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9553{
9554 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9555 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9556
9557 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9558 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9559 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9560 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9561 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9562 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9563 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9564 AssertRCReturn(rc2, rc2);
9565
9566 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9567 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9568 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9569 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9570 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9571 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9572 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9573
9574 /* I/O operation lookup arrays. */
9575 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9576 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9577
9578 VBOXSTRICTRC rcStrict;
9579 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9580 const uint32_t cbInstr = pVmxTransient->cbInstr;
9581 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9582 PVM pVM = pVCpu->CTX_SUFF(pVM);
9583 if (fIOString)
9584 {
9585 /*
9586 * INS/OUTS - I/O String instruction.
9587 *
9588 * Use instruction-information if available, otherwise fall back on
9589 * interpreting the instruction.
9590 */
9591 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9592#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9593 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9594 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9595 {
9596 rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9597 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9598 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9599 AssertRCReturn(rc2, rc2);
9600 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9601 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9602 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9603 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9604 if (fIOWrite)
9605 {
9606 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9607 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9608 //if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9609 // hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9610 // pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9611 }
9612 else
9613 {
9614 AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9615 ("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9616 VERR_HMVMX_IPE_4);
9617 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9618 //if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9619 // hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9620 }
9621 }
9622 else
9623 {
9624 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9625 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9626 AssertRCReturn(rc2, rc2);
9627 rcStrict = IEMExecOne(pVCpu);
9628 }
9629 /** @todo IEM needs to be setting these flags somehow. */
9630 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9631 fUpdateRipAlready = true;
9632#else
9633 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9634 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9635 if (RT_SUCCESS(rcStrict))
9636 {
9637 if (fIOWrite)
9638 {
9639 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9640 (DISCPUMODE)pDis->uAddrMode, cbValue);
9641 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9642 }
9643 else
9644 {
9645 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9646 (DISCPUMODE)pDis->uAddrMode, cbValue);
9647 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9648 }
9649 }
9650 else
9651 {
9652 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9653 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9654 }
9655#endif
9656 }
9657 else
9658 {
9659 /*
9660 * IN/OUT - I/O instruction.
9661 */
9662 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9663 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9664 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9665 if (fIOWrite)
9666 {
9667 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9668 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9669 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9670 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9671 }
9672 else
9673 {
9674 uint32_t u32Result = 0;
9675 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9676 if (IOM_SUCCESS(rcStrict))
9677 {
9678 /* Save result of I/O IN instr. in AL/AX/EAX. */
9679 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
9680 }
9681 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9682 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9683 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9684 }
9685 }
9686
9687 if (IOM_SUCCESS(rcStrict))
9688 {
9689 if (!fUpdateRipAlready)
9690 {
9691 pMixedCtx->rip += cbInstr;
9692 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9693 }
9694
9695 /*
9696 * If any I/O breakpoints are armed, we need to check if one triggered
9697 * and take appropriate action.
9698 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9699 */
9700 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9701 AssertRCReturn(rc2, rc2);
9702
9703 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9704 * execution engines about whether hyper BPs and such are pending. */
9705 uint32_t const uDr7 = pMixedCtx->dr[7];
9706 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9707 && X86_DR7_ANY_RW_IO(uDr7)
9708 && (pMixedCtx->cr4 & X86_CR4_DE))
9709 || DBGFBpIsHwIoArmed(pVM)))
9710 {
9711 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9712
9713 /* We're playing with the host CPU state here, make sure we don't preempt. */
9714 HM_DISABLE_PREEMPT_IF_NEEDED();
9715 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
9716
9717 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9718 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9719 {
9720 /* Raise #DB. */
9721 if (fIsGuestDbgActive)
9722 ASMSetDR6(pMixedCtx->dr[6]);
9723 if (pMixedCtx->dr[7] != uDr7)
9724 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9725
9726 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9727 }
9728 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9729 else if ( rcStrict2 != VINF_SUCCESS
9730 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
9731 rcStrict = rcStrict2;
9732
9733 HM_RESTORE_PREEMPT_IF_NEEDED();
9734 }
9735 }
9736
9737#ifdef DEBUG
9738 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9739 Assert(!fIOWrite);
9740 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9741 Assert(fIOWrite);
9742 else
9743 {
9744 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9745 * statuses, that the VMM device and some others may return. See
9746 * IOM_SUCCESS() for guidance. */
9747 AssertMsg( RT_FAILURE(rcStrict)
9748 || rcStrict == VINF_SUCCESS
9749 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
9750 || rcStrict == VINF_EM_DBG_BREAKPOINT
9751 || rcStrict == VINF_EM_RAW_GUEST_TRAP
9752 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9753 }
9754#endif
9755
9756 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9757 return VBOXSTRICTRC_TODO(rcStrict);
9758}
9759
9760
9761/**
9762 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9763 * VM-exit.
9764 */
9765HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9766{
9767 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9768
9769 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
9770 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9771 AssertRCReturn(rc, rc);
9772 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9773 {
9774 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9775 AssertRCReturn(rc, rc);
9776 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9777 {
9778 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9779
9780 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9781 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9782 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9783 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9784 {
9785 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9786 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9787
9788 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9789 Assert(!pVCpu->hm.s.Event.fPending);
9790 pVCpu->hm.s.Event.fPending = true;
9791 pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9792 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9793 AssertRCReturn(rc, rc);
9794 if (fErrorCodeValid)
9795 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9796 else
9797 pVCpu->hm.s.Event.u32ErrCode = 0;
9798 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9799 && uVector == X86_XCPT_PF)
9800 {
9801 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9802 }
9803
9804 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9805 }
9806 }
9807 }
9808
9809 /** @todo Emulate task switch someday, currently just going back to ring-3 for
9810 * emulation. */
9811 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9812 return VERR_EM_INTERPRETER;
9813}
9814
9815
9816/**
9817 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9818 */
9819HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9820{
9821 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9822 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9823 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9824 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9825 AssertRCReturn(rc, rc);
9826 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9827 return VINF_EM_DBG_STEPPED;
9828}
9829
9830
9831/**
9832 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9833 */
9834HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9835{
9836 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9837
9838 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9839 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9840 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9841 return VINF_SUCCESS;
9842 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9843 return rc;
9844
9845#if 0
9846 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9847 * just sync the whole thing. */
9848 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9849#else
9850 /* Aggressive state sync. for now. */
9851 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9852 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9853 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9854#endif
9855 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9856 AssertRCReturn(rc, rc);
9857
9858 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9859 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9860 switch (uAccessType)
9861 {
9862 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9863 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9864 {
9865 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9866 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9867 {
9868 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9869 }
9870
9871 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9872 GCPhys &= PAGE_BASE_GC_MASK;
9873 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9874 PVM pVM = pVCpu->CTX_SUFF(pVM);
9875 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9876 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9877
9878 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9879 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9880 CPUMCTX2CORE(pMixedCtx), GCPhys);
9881 rc = VBOXSTRICTRC_VAL(rc2);
9882 Log4(("ApicAccess rc=%d\n", rc));
9883 if ( rc == VINF_SUCCESS
9884 || rc == VERR_PAGE_TABLE_NOT_PRESENT
9885 || rc == VERR_PAGE_NOT_PRESENT)
9886 {
9887 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
9888 | HM_CHANGED_VMX_GUEST_APIC_STATE;
9889 rc = VINF_SUCCESS;
9890 }
9891 break;
9892 }
9893
9894 default:
9895 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9896 rc = VINF_EM_RAW_EMULATE_INSTR;
9897 break;
9898 }
9899
9900 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9901 return rc;
9902}
9903
9904
9905/**
9906 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9907 * VM-exit.
9908 */
9909HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9910{
9911 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9912
9913 /* We should -not- get this VM-exit if the guest is debugging. */
9914 if (CPUMIsGuestDebugStateActive(pVCpu))
9915 {
9916 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9917 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9918 }
9919
9920 int rc = VERR_INTERNAL_ERROR_5;
9921 if ( !DBGFIsStepping(pVCpu)
9922 && !pVCpu->hm.s.fSingleInstruction
9923 && !CPUMIsHyperDebugStateActive(pVCpu))
9924 {
9925 /* Don't intercept MOV DRx and #DB any more. */
9926 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
9927 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9928 AssertRCReturn(rc, rc);
9929
9930 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9931 {
9932#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9933 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
9934 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9935 AssertRCReturn(rc, rc);
9936#endif
9937 }
9938
9939 /* We're playing with the host CPU state here, make sure we can't preempt. */
9940 HM_DISABLE_PREEMPT_IF_NEEDED();
9941
9942 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9943 PVM pVM = pVCpu->CTX_SUFF(pVM);
9944 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9945 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
9946
9947 HM_RESTORE_PREEMPT_IF_NEEDED();
9948
9949#ifdef VBOX_WITH_STATISTICS
9950 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9951 AssertRCReturn(rc, rc);
9952 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9953 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9954 else
9955 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9956#endif
9957 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
9958 return VINF_SUCCESS;
9959 }
9960
9961 /*
9962 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
9963 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
9964 */
9965 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9966 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9967 AssertRCReturn(rc, rc);
9968 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9969
9970 PVM pVM = pVCpu->CTX_SUFF(pVM);
9971 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9972 {
9973 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9974 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
9975 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
9976 if (RT_SUCCESS(rc))
9977 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9978 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9979 }
9980 else
9981 {
9982 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9983 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
9984 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
9985 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9986 }
9987
9988 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9989 if (RT_SUCCESS(rc))
9990 {
9991 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9992 AssertRCReturn(rc2, rc2);
9993 }
9994 return rc;
9995}
9996
9997
9998/**
9999 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
10000 * Conditional VM-exit.
10001 */
10002HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10003{
10004 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10005 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10006
10007 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10008 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10009 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10010 return VINF_SUCCESS;
10011 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10012 return rc;
10013
10014 RTGCPHYS GCPhys = 0;
10015 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10016
10017#if 0
10018 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10019#else
10020 /* Aggressive state sync. for now. */
10021 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10022 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10023 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10024#endif
10025 AssertRCReturn(rc, rc);
10026
10027 /*
10028 * If we succeed, resume guest execution.
10029 * If we fail in interpreting the instruction because we couldn't get the guest physical address
10030 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
10031 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
10032 * weird case. See @bugref{6043}.
10033 */
10034 PVM pVM = pVCpu->CTX_SUFF(pVM);
10035 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10036 rc = VBOXSTRICTRC_VAL(rc2);
10037 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10038 if ( rc == VINF_SUCCESS
10039 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10040 || rc == VERR_PAGE_NOT_PRESENT)
10041 {
10042 /* Successfully handled MMIO operation. */
10043 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10044 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10045 rc = VINF_SUCCESS;
10046 }
10047 return rc;
10048}
10049
10050
10051/**
10052 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10053 * VM-exit.
10054 */
10055HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10056{
10057 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10058 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10059
10060 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10061 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10062 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10063 return VINF_SUCCESS;
10064 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10065 return rc;
10066
10067 RTGCPHYS GCPhys = 0;
10068 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10069 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10070#if 0
10071 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10072#else
10073 /* Aggressive state sync. for now. */
10074 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10075 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10076 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10077#endif
10078 AssertRCReturn(rc, rc);
10079
10080 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10081 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10082
10083 RTGCUINT uErrorCode = 0;
10084 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10085 uErrorCode |= X86_TRAP_PF_ID;
10086 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10087 uErrorCode |= X86_TRAP_PF_RW;
10088 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10089 uErrorCode |= X86_TRAP_PF_P;
10090
10091 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10092
10093 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10094 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10095
10096 /* Handle the pagefault trap for the nested shadow table. */
10097 PVM pVM = pVCpu->CTX_SUFF(pVM);
10098 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10099 TRPMResetTrap(pVCpu);
10100
10101 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10102 if ( rc == VINF_SUCCESS
10103 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10104 || rc == VERR_PAGE_NOT_PRESENT)
10105 {
10106 /* Successfully synced our nested page tables. */
10107 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10108 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10109 return VINF_SUCCESS;
10110 }
10111
10112 Log4(("EPT return to ring-3 rc=%d\n"));
10113 return rc;
10114}
10115
10116/** @} */
10117
10118/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10119/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10120/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10121
10122/** @name VM-exit exception handlers.
10123 * @{
10124 */
10125
10126/**
10127 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10128 */
10129static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10130{
10131 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10132 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10133
10134 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10135 AssertRCReturn(rc, rc);
10136
10137 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10138 {
10139 /* Old-style FPU error reporting needs some extra work. */
10140 /** @todo don't fall back to the recompiler, but do it manually. */
10141 return VERR_EM_INTERPRETER;
10142 }
10143
10144 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10145 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10146 return rc;
10147}
10148
10149
10150/**
10151 * VM-exit exception handler for #BP (Breakpoint exception).
10152 */
10153static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10154{
10155 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10156 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10157
10158 /** @todo Try optimize this by not saving the entire guest state unless
10159 * really needed. */
10160 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10161 AssertRCReturn(rc, rc);
10162
10163 PVM pVM = pVCpu->CTX_SUFF(pVM);
10164 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10165 if (rc == VINF_EM_RAW_GUEST_TRAP)
10166 {
10167 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10168 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10169 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10170 AssertRCReturn(rc, rc);
10171
10172 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10173 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10174 }
10175
10176 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10177 return rc;
10178}
10179
10180
10181/**
10182 * VM-exit exception handler for #DB (Debug exception).
10183 */
10184static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10185{
10186 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10187 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10188 Log6(("XcptDB\n"));
10189
10190 /*
10191 * Get the DR6-like values from the exit qualification and pass it to DBGF
10192 * for processing.
10193 */
10194 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10195 AssertRCReturn(rc, rc);
10196
10197 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10198 uint64_t uDR6 = X86_DR6_INIT_VAL;
10199 uDR6 |= ( pVmxTransient->uExitQualification
10200 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10201
10202 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10203 if (rc == VINF_EM_RAW_GUEST_TRAP)
10204 {
10205 /*
10206 * The exception was for the guest. Update DR6, DR7.GD and
10207 * IA32_DEBUGCTL.LBR before forwarding it.
10208 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10209 */
10210 HM_DISABLE_PREEMPT_IF_NEEDED();
10211
10212 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10213 pMixedCtx->dr[6] |= uDR6;
10214 if (CPUMIsGuestDebugStateActive(pVCpu))
10215 ASMSetDR6(pMixedCtx->dr[6]);
10216
10217 HM_RESTORE_PREEMPT_IF_NEEDED();
10218
10219 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10220 AssertRCReturn(rc, rc);
10221
10222 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10223 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10224
10225 /* Paranoia. */
10226 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10227 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10228
10229 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10230 AssertRCReturn(rc, rc);
10231
10232 /*
10233 * Raise #DB in the guest.
10234 */
10235 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10236 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10237 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10238 AssertRCReturn(rc, rc);
10239 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10240 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10241 return VINF_SUCCESS;
10242 }
10243
10244 /*
10245 * Not a guest trap, must be a hypervisor related debug event then.
10246 * Update DR6 in case someone is interested in it.
10247 */
10248 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10249 AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
10250 CPUMSetHyperDR6(pVCpu, uDR6);
10251
10252 return rc;
10253}
10254
10255
10256/**
10257 * VM-exit exception handler for #NM (Device-not-available exception: floating
10258 * point exception).
10259 */
10260static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10261{
10262 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10263
10264#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10265 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
10266#endif
10267
10268 /* We require CR0 and EFER. EFER is always up-to-date. */
10269 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10270 AssertRCReturn(rc, rc);
10271
10272 /* We're playing with the host CPU state here, have to disable preemption. */
10273 HM_DISABLE_PREEMPT_IF_NEEDED();
10274
10275 /* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10276 PVM pVM = pVCpu->CTX_SUFF(pVM);
10277 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
10278 if (rc == VINF_SUCCESS)
10279 {
10280 Assert(CPUMIsGuestFPUStateActive(pVCpu));
10281 HM_RESTORE_PREEMPT_IF_NEEDED();
10282
10283 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
10284 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10285 return VINF_SUCCESS;
10286 }
10287
10288 HM_RESTORE_PREEMPT_IF_NEEDED();
10289
10290 /* Forward #NM to the guest. */
10291 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10292 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10293 AssertRCReturn(rc, rc);
10294 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10295 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10296 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10297 return rc;
10298}
10299
10300
10301/**
10302 * VM-exit exception handler for #GP (General-protection exception).
10303 *
10304 * @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
10305 */
10306static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10307{
10308 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10309 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10310
10311 int rc = VERR_INTERNAL_ERROR_5;
10312 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10313 {
10314#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10315 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10316 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10317 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10318 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10319 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10320 AssertRCReturn(rc, rc);
10321 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10322 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10323 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10324 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10325 return rc;
10326#else
10327 /* We don't intercept #GP. */
10328 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10329 return VERR_VMX_UNEXPECTED_EXCEPTION;
10330#endif
10331 }
10332
10333 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10334 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10335
10336 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10337 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10338 AssertRCReturn(rc, rc);
10339
10340 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10341 uint32_t cbOp = 0;
10342 PVM pVM = pVCpu->CTX_SUFF(pVM);
10343 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10344 if (RT_SUCCESS(rc))
10345 {
10346 rc = VINF_SUCCESS;
10347 Assert(cbOp == pDis->cbInstr);
10348 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10349 switch (pDis->pCurInstr->uOpcode)
10350 {
10351 case OP_CLI:
10352 {
10353 pMixedCtx->eflags.Bits.u1IF = 0;
10354 pMixedCtx->rip += pDis->cbInstr;
10355 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10357 break;
10358 }
10359
10360 case OP_STI:
10361 {
10362 pMixedCtx->eflags.Bits.u1IF = 1;
10363 pMixedCtx->rip += pDis->cbInstr;
10364 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10365 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10366 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10367 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10368 break;
10369 }
10370
10371 case OP_HLT:
10372 {
10373 rc = VINF_EM_HALT;
10374 pMixedCtx->rip += pDis->cbInstr;
10375 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
10376 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10377 break;
10378 }
10379
10380 case OP_POPF:
10381 {
10382 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10383 uint32_t cbParm = 0;
10384 uint32_t uMask = 0;
10385 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10386 {
10387 cbParm = 4;
10388 uMask = 0xffffffff;
10389 }
10390 else
10391 {
10392 cbParm = 2;
10393 uMask = 0xffff;
10394 }
10395
10396 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10397 RTGCPTR GCPtrStack = 0;
10398 X86EFLAGS Eflags;
10399 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10400 &GCPtrStack);
10401 if (RT_SUCCESS(rc))
10402 {
10403 Assert(sizeof(Eflags.u32) >= cbParm);
10404 Eflags.u32 = 0;
10405 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10406 }
10407 if (RT_FAILURE(rc))
10408 {
10409 rc = VERR_EM_INTERPRETER;
10410 break;
10411 }
10412 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10413 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10414 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10415 /* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10416 pMixedCtx->eflags.Bits.u1RF = 0;
10417 pMixedCtx->esp += cbParm;
10418 pMixedCtx->esp &= uMask;
10419 pMixedCtx->rip += pDis->cbInstr;
10420 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10421 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10422 break;
10423 }
10424
10425 case OP_PUSHF:
10426 {
10427 uint32_t cbParm = 0;
10428 uint32_t uMask = 0;
10429 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10430 {
10431 cbParm = 4;
10432 uMask = 0xffffffff;
10433 }
10434 else
10435 {
10436 cbParm = 2;
10437 uMask = 0xffff;
10438 }
10439
10440 /* Get the stack pointer & push the contents of eflags onto the stack. */
10441 RTGCPTR GCPtrStack = 0;
10442 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10443 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10444 if (RT_FAILURE(rc))
10445 {
10446 rc = VERR_EM_INTERPRETER;
10447 break;
10448 }
10449 X86EFLAGS Eflags = pMixedCtx->eflags;
10450 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10451 Eflags.Bits.u1RF = 0;
10452 Eflags.Bits.u1VM = 0;
10453
10454 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10455 if (RT_FAILURE(rc))
10456 {
10457 rc = VERR_EM_INTERPRETER;
10458 break;
10459 }
10460 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10461 pMixedCtx->esp -= cbParm;
10462 pMixedCtx->esp &= uMask;
10463 pMixedCtx->rip += pDis->cbInstr;
10464 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP;
10465 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10466 break;
10467 }
10468
10469 case OP_IRET:
10470 {
10471 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10472 * instruction reference. */
10473 RTGCPTR GCPtrStack = 0;
10474 uint32_t uMask = 0xffff;
10475 uint16_t aIretFrame[3];
10476 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10477 {
10478 rc = VERR_EM_INTERPRETER;
10479 break;
10480 }
10481 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10482 &GCPtrStack);
10483 if (RT_SUCCESS(rc))
10484 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10485 if (RT_FAILURE(rc))
10486 {
10487 rc = VERR_EM_INTERPRETER;
10488 break;
10489 }
10490 pMixedCtx->eip = 0;
10491 pMixedCtx->ip = aIretFrame[0];
10492 pMixedCtx->cs.Sel = aIretFrame[1];
10493 pMixedCtx->cs.ValidSel = aIretFrame[1];
10494 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10495 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10496 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10497 pMixedCtx->sp += sizeof(aIretFrame);
10498 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_SEGMENT_REGS | HM_CHANGED_GUEST_RSP
10499 | HM_CHANGED_GUEST_RFLAGS;
10500 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10501 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10502 break;
10503 }
10504
10505 case OP_INT:
10506 {
10507 uint16_t uVector = pDis->Param1.uValue & 0xff;
10508 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10509 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10510 break;
10511 }
10512
10513 case OP_INTO:
10514 {
10515 if (pMixedCtx->eflags.Bits.u1OF)
10516 {
10517 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10518 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10519 }
10520 break;
10521 }
10522
10523 default:
10524 {
10525 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10526 EMCODETYPE_SUPERVISOR);
10527 rc = VBOXSTRICTRC_VAL(rc2);
10528 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
10529 Log4(("#GP rc=%Rrc\n", rc));
10530 break;
10531 }
10532 }
10533 }
10534 else
10535 rc = VERR_EM_INTERPRETER;
10536
10537 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10538 ("#GP Unexpected rc=%Rrc\n", rc));
10539 return rc;
10540}
10541
10542
10543/**
10544 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10545 * the exception reported in the VMX transient structure back into the VM.
10546 *
10547 * @remarks Requires uExitIntrInfo in the VMX transient structure to be
10548 * up-to-date.
10549 */
10550static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10551{
10552 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10553
10554 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10555 hmR0VmxCheckExitDueToEventDelivery(). */
10556 int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10557 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10558 AssertRCReturn(rc, rc);
10559 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10560
10561 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10562 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10563 return VINF_SUCCESS;
10564}
10565
10566
10567/**
10568 * VM-exit exception handler for #PF (Page-fault exception).
10569 */
10570static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10571{
10572 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10573 PVM pVM = pVCpu->CTX_SUFF(pVM);
10574 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10575 rc |= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10576 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10577 AssertRCReturn(rc, rc);
10578
10579#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
10580 if (pVM->hm.s.fNestedPaging)
10581 {
10582 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10583 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10584 {
10585 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10586 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10587 0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10588 }
10589 else
10590 {
10591 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10592 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10593 Log4(("Pending #DF due to vectoring #PF. NP\n"));
10594 }
10595 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10596 return rc;
10597 }
10598#else
10599 Assert(!pVM->hm.s.fNestedPaging);
10600#endif
10601
10602 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10603 AssertRCReturn(rc, rc);
10604
10605 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10606 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10607
10608 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10609 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10610 (RTGCPTR)pVmxTransient->uExitQualification);
10611
10612 Log4(("#PF: rc=%Rrc\n", rc));
10613 if (rc == VINF_SUCCESS)
10614 {
10615 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10616 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
10617 * memory? We don't update the whole state here... */
10618 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10619 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10620 TRPMResetTrap(pVCpu);
10621 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10622 return rc;
10623 }
10624 else if (rc == VINF_EM_RAW_GUEST_TRAP)
10625 {
10626 if (!pVmxTransient->fVectoringPF)
10627 {
10628 /* It's a guest page fault and needs to be reflected to the guest. */
10629 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10630 TRPMResetTrap(pVCpu);
10631 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10632 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10633 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10634 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10635 }
10636 else
10637 {
10638 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10639 TRPMResetTrap(pVCpu);
10640 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10641 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10642 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10643 }
10644
10645 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10646 return VINF_SUCCESS;
10647 }
10648
10649 TRPMResetTrap(pVCpu);
10650 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10651 return rc;
10652}
10653
10654/** @} */
10655
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette