VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 48248

Last change on this file since 48248 was 48248, checked in by vboxsync, 12 years ago

Comments.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 445.1 KB
Line 
1/* $Id: HMVMXR0.cpp 48248 2013-09-03 12:44:22Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#endif
45
46
47/*******************************************************************************
48* Defined Constants And Macros *
49*******************************************************************************/
50#if defined(RT_ARCH_AMD64)
51# define HMVMX_IS_64BIT_HOST_MODE() (true)
52typedef RTHCUINTREG HMVMXHCUINTREG;
53#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54extern "C" uint32_t g_fVMXIs64bitHost;
55# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56typedef uint64_t HMVMXHCUINTREG;
57#else
58# define HMVMX_IS_64BIT_HOST_MODE() (false)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#endif
61
62/** Use the function table. */
63#define HMVMX_USE_FUNCTION_TABLE
64
65/** Determine which tagged-TLB flush handler to use. */
66#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71/** @name Updated-guest-state flags.
72 * @{ */
73#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94 | HMVMX_UPDATED_GUEST_RSP \
95 | HMVMX_UPDATED_GUEST_RFLAGS \
96 | HMVMX_UPDATED_GUEST_CR0 \
97 | HMVMX_UPDATED_GUEST_CR3 \
98 | HMVMX_UPDATED_GUEST_CR4 \
99 | HMVMX_UPDATED_GUEST_GDTR \
100 | HMVMX_UPDATED_GUEST_IDTR \
101 | HMVMX_UPDATED_GUEST_LDTR \
102 | HMVMX_UPDATED_GUEST_TR \
103 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104 | HMVMX_UPDATED_GUEST_DEBUG \
105 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112 | HMVMX_UPDATED_GUEST_APIC_STATE)
113/** @} */
114
115/** @name
116 * Flags to skip redundant reads of some common VMCS fields that are not part of
117 * the guest-CPU state but are in the transient structure.
118 */
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually (except #PF). #NM is also
143 * handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
144 * intercepted even in real-mode if we have Nested Paging support.
145 */
146#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
147 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
148 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
149 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
150 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
151 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
152 | RT_BIT(X86_XCPT_XF))
153
154/**
155 * Exception bitmap mask for all contributory exceptions.
156 *
157 * Page fault is deliberately excluded here as it's conditional as to whether
158 * it's contributory or benign. Page faults are handled separately.
159 */
160#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
161 | RT_BIT(X86_XCPT_DE))
162
163/** Maximum VM-instruction error number. */
164#define HMVMX_INSTR_ERROR_MAX 28
165
166/** Profiling macro. */
167#ifdef HM_PROFILE_EXIT_DISPATCH
168# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
169# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
170#else
171# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
173#endif
174
175/** Assert that preemption is disabled or covered by thread-context hooks. */
176#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
177 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
178
179/** Assert that we haven't migrated CPUs when thread-context hooks are not
180 * used. */
181#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
182 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
183 ("Illegal migration! Entered on CPU %u Current %u\n", \
184 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
185
186/*******************************************************************************
187* Structures and Typedefs *
188*******************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG uEflags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
203 uint64_t u64LStarMsr;
204 /** The guest's TPR value used for TPR shadowing. */
205 uint8_t u8GuestTpr;
206 /** Alignment. */
207 uint8_t abAlignment0[7];
208
209 /** The basic VM-exit reason. */
210 uint16_t uExitReason;
211 /** Alignment. */
212 uint16_t u16Alignment0;
213 /** The VM-exit interruption error code. */
214 uint32_t uExitIntrErrorCode;
215 /** The VM-exit exit qualification. */
216 uint64_t uExitQualification;
217
218 /** The VM-exit interruption-information field. */
219 uint32_t uExitIntrInfo;
220 /** The VM-exit instruction-length field. */
221 uint32_t cbInstr;
222 /** The VM-exit instruction-information field. */
223 union
224 {
225 /** Plain unsigned int representation. */
226 uint32_t u;
227 /** INS and OUTS information. */
228 struct
229 {
230 uint32_t u6Reserved0 : 6;
231 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
232 uint32_t u3AddrSize : 3;
233 uint32_t u5Reserved1 : 5;
234 /** The segment register (X86_SREG_XXX). */
235 uint32_t iSegReg : 3;
236 uint32_t uReserved2 : 14;
237 } StrIo;
238 } ExitInstrInfo;
239 /** Whether the VM-entry failed or not. */
240 bool fVMEntryFailed;
241 /** Alignment. */
242 uint8_t abAlignment1[3];
243
244 /** The VM-entry interruption-information field. */
245 uint32_t uEntryIntrInfo;
246 /** The VM-entry exception error code field. */
247 uint32_t uEntryXcptErrorCode;
248 /** The VM-entry instruction length field. */
249 uint32_t cbEntryInstr;
250
251 /** IDT-vectoring information field. */
252 uint32_t uIdtVectoringInfo;
253 /** IDT-vectoring error code. */
254 uint32_t uIdtVectoringErrorCode;
255
256 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
257 uint32_t fVmcsFieldsRead;
258 /** Whether TSC-offsetting should be setup before VM-entry. */
259 bool fUpdateTscOffsettingAndPreemptTimer;
260 /** Whether the VM-exit was caused by a page-fault during delivery of a
261 * contributory exception or a page-fault. */
262 bool fVectoringPF;
263} VMXTRANSIENT;
264AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
265AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
266AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
267AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
268/** Pointer to VMX transient state. */
269typedef VMXTRANSIENT *PVMXTRANSIENT;
270
271
272/**
273 * MSR-bitmap read permissions.
274 */
275typedef enum VMXMSREXITREAD
276{
277 /** Reading this MSR causes a VM-exit. */
278 VMXMSREXIT_INTERCEPT_READ = 0xb,
279 /** Reading this MSR does not cause a VM-exit. */
280 VMXMSREXIT_PASSTHRU_READ
281} VMXMSREXITREAD;
282
283/**
284 * MSR-bitmap write permissions.
285 */
286typedef enum VMXMSREXITWRITE
287{
288 /** Writing to this MSR causes a VM-exit. */
289 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
290 /** Writing to this MSR does not cause a VM-exit. */
291 VMXMSREXIT_PASSTHRU_WRITE
292} VMXMSREXITWRITE;
293
294/**
295 * VM-exit handler.
296 *
297 * @returns VBox status code.
298 * @param pVCpu Pointer to the VMCPU.
299 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
300 * out-of-sync. Make sure to update the required
301 * fields before using them.
302 * @param pVmxTransient Pointer to the VMX-transient structure.
303 */
304#ifndef HMVMX_USE_FUNCTION_TABLE
305typedef int FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306#else
307typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308/** Pointer to VM-exit handler. */
309typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
310#endif
311
312
313/*******************************************************************************
314* Internal Functions *
315*******************************************************************************/
316static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
317static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
318static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
319 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
320#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
321static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
322#endif
323#ifndef HMVMX_USE_FUNCTION_TABLE
324DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
325# define HMVMX_EXIT_DECL static int
326#else
327# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
328#endif
329
330/** @name VM-exit handlers.
331 * @{
332 */
333static FNVMEXITHANDLER hmR0VmxExitXcptOrNmi;
334static FNVMEXITHANDLER hmR0VmxExitExtInt;
335static FNVMEXITHANDLER hmR0VmxExitTripleFault;
336static FNVMEXITHANDLER hmR0VmxExitInitSignal;
337static FNVMEXITHANDLER hmR0VmxExitSipi;
338static FNVMEXITHANDLER hmR0VmxExitIoSmi;
339static FNVMEXITHANDLER hmR0VmxExitSmi;
340static FNVMEXITHANDLER hmR0VmxExitIntWindow;
341static FNVMEXITHANDLER hmR0VmxExitNmiWindow;
342static FNVMEXITHANDLER hmR0VmxExitTaskSwitch;
343static FNVMEXITHANDLER hmR0VmxExitCpuid;
344static FNVMEXITHANDLER hmR0VmxExitGetsec;
345static FNVMEXITHANDLER hmR0VmxExitHlt;
346static FNVMEXITHANDLER hmR0VmxExitInvd;
347static FNVMEXITHANDLER hmR0VmxExitInvlpg;
348static FNVMEXITHANDLER hmR0VmxExitRdpmc;
349static FNVMEXITHANDLER hmR0VmxExitRdtsc;
350static FNVMEXITHANDLER hmR0VmxExitRsm;
351static FNVMEXITHANDLER hmR0VmxExitSetPendingXcptUD;
352static FNVMEXITHANDLER hmR0VmxExitMovCRx;
353static FNVMEXITHANDLER hmR0VmxExitMovDRx;
354static FNVMEXITHANDLER hmR0VmxExitIoInstr;
355static FNVMEXITHANDLER hmR0VmxExitRdmsr;
356static FNVMEXITHANDLER hmR0VmxExitWrmsr;
357static FNVMEXITHANDLER hmR0VmxExitErrInvalidGuestState;
358static FNVMEXITHANDLER hmR0VmxExitErrMsrLoad;
359static FNVMEXITHANDLER hmR0VmxExitErrUndefined;
360static FNVMEXITHANDLER hmR0VmxExitMwait;
361static FNVMEXITHANDLER hmR0VmxExitMtf;
362static FNVMEXITHANDLER hmR0VmxExitMonitor;
363static FNVMEXITHANDLER hmR0VmxExitPause;
364static FNVMEXITHANDLER hmR0VmxExitErrMachineCheck;
365static FNVMEXITHANDLER hmR0VmxExitTprBelowThreshold;
366static FNVMEXITHANDLER hmR0VmxExitApicAccess;
367static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
368static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
369static FNVMEXITHANDLER hmR0VmxExitEptViolation;
370static FNVMEXITHANDLER hmR0VmxExitEptMisconfig;
371static FNVMEXITHANDLER hmR0VmxExitRdtscp;
372static FNVMEXITHANDLER hmR0VmxExitPreemptTimer;
373static FNVMEXITHANDLER hmR0VmxExitWbinvd;
374static FNVMEXITHANDLER hmR0VmxExitXsetbv;
375static FNVMEXITHANDLER hmR0VmxExitRdrand;
376static FNVMEXITHANDLER hmR0VmxExitInvpcid;
377/** @} */
378
379static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
380static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
381static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
382static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
383static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
384static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
385static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
386static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
387
388/*******************************************************************************
389* Global Variables *
390*******************************************************************************/
391#ifdef HMVMX_USE_FUNCTION_TABLE
392
393/**
394 * VMX_EXIT dispatch table.
395 */
396static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
397{
398 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
399 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
400 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
401 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
402 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
403 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
404 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
405 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
406 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
407 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
408 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
409 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
410 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
411 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
412 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
413 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
414 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
415 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
416 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
417 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
418 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
419 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
420 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
421 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
422 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
423 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
424 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
425 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
426 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
427 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
428 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
429 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
430 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
431 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
432 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
433 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
434 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
435 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
436 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
437 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
438 /* 40 UNDEFINED */ hmR0VmxExitPause,
439 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
440 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
441 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
442 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
443 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
444 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
445 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
446 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
447 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
448 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
449 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
450 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
451 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
452 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
453 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
454 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
455 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
456 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
457 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
458};
459#endif /* HMVMX_USE_FUNCTION_TABLE */
460
461#ifdef VBOX_STRICT
462static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
463{
464 /* 0 */ "(Not Used)",
465 /* 1 */ "VMCALL executed in VMX root operation.",
466 /* 2 */ "VMCLEAR with invalid physical address.",
467 /* 3 */ "VMCLEAR with VMXON pointer.",
468 /* 4 */ "VMLAUNCH with non-clear VMCS.",
469 /* 5 */ "VMRESUME with non-launched VMCS.",
470 /* 6 */ "VMRESUME after VMXOFF",
471 /* 7 */ "VM entry with invalid control fields.",
472 /* 8 */ "VM entry with invalid host state fields.",
473 /* 9 */ "VMPTRLD with invalid physical address.",
474 /* 10 */ "VMPTRLD with VMXON pointer.",
475 /* 11 */ "VMPTRLD with incorrect revision identifier.",
476 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
477 /* 13 */ "VMWRITE to read-only VMCS component.",
478 /* 14 */ "(Not Used)",
479 /* 15 */ "VMXON executed in VMX root operation.",
480 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
481 /* 17 */ "VM entry with non-launched executing VMCS.",
482 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
483 /* 19 */ "VMCALL with non-clear VMCS.",
484 /* 20 */ "VMCALL with invalid VM-exit control fields.",
485 /* 21 */ "(Not Used)",
486 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
487 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
488 /* 24 */ "VMCALL with invalid SMM-monitor features.",
489 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
490 /* 26 */ "VM entry with events blocked by MOV SS.",
491 /* 27 */ "(Not Used)",
492 /* 28 */ "Invalid operand to INVEPT/INVVPID."
493};
494#endif /* VBOX_STRICT */
495
496
497
498/**
499 * Updates the VM's last error record. If there was a VMX instruction error,
500 * reads the error data from the VMCS and updates VCPU's last error record as
501 * well.
502 *
503 * @param pVM Pointer to the VM.
504 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
505 * VERR_VMX_UNABLE_TO_START_VM or
506 * VERR_VMX_INVALID_VMCS_FIELD).
507 * @param rc The error code.
508 */
509static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
510{
511 AssertPtr(pVM);
512 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
513 || rc == VERR_VMX_UNABLE_TO_START_VM)
514 {
515 AssertPtrReturnVoid(pVCpu);
516 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
517 }
518 pVM->hm.s.lLastError = rc;
519}
520
521
522/**
523 * Reads the VM-entry interruption-information field from the VMCS into the VMX
524 * transient structure.
525 *
526 * @returns VBox status code.
527 * @param pVmxTransient Pointer to the VMX transient structure.
528 *
529 * @remarks No-long-jump zone!!!
530 */
531DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
532{
533 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
534 AssertRCReturn(rc, rc);
535 return VINF_SUCCESS;
536}
537
538
539/**
540 * Reads the VM-entry exception error code field from the VMCS into
541 * the VMX transient structure.
542 *
543 * @returns VBox status code.
544 * @param pVmxTransient Pointer to the VMX transient structure.
545 *
546 * @remarks No-long-jump zone!!!
547 */
548DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
549{
550 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
551 AssertRCReturn(rc, rc);
552 return VINF_SUCCESS;
553}
554
555
556/**
557 * Reads the VM-entry exception error code field from the VMCS into
558 * the VMX transient structure.
559 *
560 * @returns VBox status code.
561 * @param pVCpu Pointer to the VMCPU.
562 * @param pVmxTransient Pointer to the VMX transient structure.
563 *
564 * @remarks No-long-jump zone!!!
565 */
566DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
567{
568 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
569 AssertRCReturn(rc, rc);
570 return VINF_SUCCESS;
571}
572
573
574/**
575 * Reads the VM-exit interruption-information field from the VMCS into the VMX
576 * transient structure.
577 *
578 * @returns VBox status code.
579 * @param pVCpu Pointer to the VMCPU.
580 * @param pVmxTransient Pointer to the VMX transient structure.
581 */
582DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
583{
584 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
585 {
586 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
587 AssertRCReturn(rc, rc);
588 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
589 }
590 return VINF_SUCCESS;
591}
592
593
594/**
595 * Reads the VM-exit interruption error code from the VMCS into the VMX
596 * transient structure.
597 *
598 * @returns VBox status code.
599 * @param pVCpu Pointer to the VMCPU.
600 * @param pVmxTransient Pointer to the VMX transient structure.
601 */
602DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
603{
604 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
605 {
606 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
607 AssertRCReturn(rc, rc);
608 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
609 }
610 return VINF_SUCCESS;
611}
612
613
614/**
615 * Reads the VM-exit instruction length field from the VMCS into the VMX
616 * transient structure.
617 *
618 * @returns VBox status code.
619 * @param pVCpu Pointer to the VMCPU.
620 * @param pVmxTransient Pointer to the VMX transient structure.
621 */
622DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
623{
624 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
625 {
626 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
627 AssertRCReturn(rc, rc);
628 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
629 }
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit instruction-information field from the VMCS into
636 * the VMX transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVCpu The cross context per CPU structure.
640 * @param pVmxTransient Pointer to the VMX transient structure.
641 */
642DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
643{
644 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645 {
646 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
647 AssertRCReturn(rc, rc);
648 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649 }
650 return VINF_SUCCESS;
651}
652
653
654/**
655 * Reads the exit qualification from the VMCS into the VMX transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVCpu Pointer to the VMCPU.
659 * @param pVmxTransient Pointer to the VMX transient structure.
660 */
661DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
662{
663 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
664 {
665 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
666 AssertRCReturn(rc, rc);
667 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
668 }
669 return VINF_SUCCESS;
670}
671
672
673/**
674 * Reads the IDT-vectoring information field from the VMCS into the VMX
675 * transient structure.
676 *
677 * @returns VBox status code.
678 * @param pVmxTransient Pointer to the VMX transient structure.
679 *
680 * @remarks No-long-jump zone!!!
681 */
682DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
683{
684 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
685 {
686 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
687 AssertRCReturn(rc, rc);
688 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
689 }
690 return VINF_SUCCESS;
691}
692
693
694/**
695 * Reads the IDT-vectoring error code from the VMCS into the VMX
696 * transient structure.
697 *
698 * @returns VBox status code.
699 * @param pVmxTransient Pointer to the VMX transient structure.
700 */
701DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
702{
703 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
704 {
705 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
706 AssertRCReturn(rc, rc);
707 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
708 }
709 return VINF_SUCCESS;
710}
711
712
713/**
714 * Enters VMX root mode operation on the current CPU.
715 *
716 * @returns VBox status code.
717 * @param pVM Pointer to the VM (optional, can be NULL, after
718 * a resume).
719 * @param HCPhysCpuPage Physical address of the VMXON region.
720 * @param pvCpuPage Pointer to the VMXON region.
721 */
722static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
723{
724 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
725 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
726 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
727
728 if (pVM)
729 {
730 /* Write the VMCS revision dword to the VMXON region. */
731 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
732 }
733
734 /* Enable the VMX bit in CR4 if necessary. */
735 RTCCUINTREG uCr4 = ASMGetCR4();
736 if (!(uCr4 & X86_CR4_VMXE))
737 ASMSetCR4(uCr4 | X86_CR4_VMXE);
738
739 /* Enter VMX root mode. */
740 int rc = VMXEnable(HCPhysCpuPage);
741 if (RT_FAILURE(rc))
742 ASMSetCR4(uCr4);
743
744 return rc;
745}
746
747
748/**
749 * Exits VMX root mode operation on the current CPU.
750 *
751 * @returns VBox status code.
752 */
753static int hmR0VmxLeaveRootMode(void)
754{
755 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
756
757 /* If we're for some reason not in VMX root mode, then don't leave it. */
758 RTCCUINTREG uHostCR4 = ASMGetCR4();
759 if (uHostCR4 & X86_CR4_VMXE)
760 {
761 /* Exit VMX root mode and clear the VMX bit in CR4. */
762 VMXDisable();
763 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
764 return VINF_SUCCESS;
765 }
766
767 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
768}
769
770
771/**
772 * Allocates and maps one physically contiguous page. The allocated page is
773 * zero'd out. (Used by various VT-x structures).
774 *
775 * @returns IPRT status code.
776 * @param pMemObj Pointer to the ring-0 memory object.
777 * @param ppVirt Where to store the virtual address of the
778 * allocation.
779 * @param pPhys Where to store the physical address of the
780 * allocation.
781 */
782DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
783{
784 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
785 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
786 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
787
788 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
789 if (RT_FAILURE(rc))
790 return rc;
791 *ppVirt = RTR0MemObjAddress(*pMemObj);
792 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
793 ASMMemZero32(*ppVirt, PAGE_SIZE);
794 return VINF_SUCCESS;
795}
796
797
798/**
799 * Frees and unmaps an allocated physical page.
800 *
801 * @param pMemObj Pointer to the ring-0 memory object.
802 * @param ppVirt Where to re-initialize the virtual address of
803 * allocation as 0.
804 * @param pHCPhys Where to re-initialize the physical address of the
805 * allocation as 0.
806 */
807DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
808{
809 AssertPtr(pMemObj);
810 AssertPtr(ppVirt);
811 AssertPtr(pHCPhys);
812 if (*pMemObj != NIL_RTR0MEMOBJ)
813 {
814 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
815 AssertRC(rc);
816 *pMemObj = NIL_RTR0MEMOBJ;
817 *ppVirt = 0;
818 *pHCPhys = 0;
819 }
820}
821
822
823/**
824 * Worker function to free VT-x related structures.
825 *
826 * @returns IPRT status code.
827 * @param pVM Pointer to the VM.
828 */
829static void hmR0VmxStructsFree(PVM pVM)
830{
831 for (VMCPUID i = 0; i < pVM->cCpus; i++)
832 {
833 PVMCPU pVCpu = &pVM->aCpus[i];
834 AssertPtr(pVCpu);
835
836#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
837 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
838 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
839#endif
840
841 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
842 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
843
844 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
845 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
846 }
847
848 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
849#ifdef VBOX_WITH_CRASHDUMP_MAGIC
850 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
851#endif
852}
853
854
855/**
856 * Worker function to allocate VT-x related VM structures.
857 *
858 * @returns IPRT status code.
859 * @param pVM Pointer to the VM.
860 */
861static int hmR0VmxStructsAlloc(PVM pVM)
862{
863 /*
864 * Initialize members up-front so we can cleanup properly on allocation failure.
865 */
866#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
867 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
868 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
869 pVM->hm.s.vmx.HCPhys##a_Name = 0;
870
871#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
872 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
873 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
874 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
875
876#ifdef VBOX_WITH_CRASHDUMP_MAGIC
877 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
878#endif
879 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
880
881 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
882 for (VMCPUID i = 0; i < pVM->cCpus; i++)
883 {
884 PVMCPU pVCpu = &pVM->aCpus[i];
885 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
886 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
887 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
888#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
889 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
890 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
891#endif
892 }
893#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
894#undef VMXLOCAL_INIT_VM_MEMOBJ
895
896 /*
897 * Allocate all the VT-x structures.
898 */
899 int rc = VINF_SUCCESS;
900#ifdef VBOX_WITH_CRASHDUMP_MAGIC
901 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
902 if (RT_FAILURE(rc))
903 goto cleanup;
904 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
905 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
906#endif
907
908 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
909 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
910 {
911 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
912 &pVM->hm.s.vmx.HCPhysApicAccess);
913 if (RT_FAILURE(rc))
914 goto cleanup;
915 }
916
917 /*
918 * Initialize per-VCPU VT-x structures.
919 */
920 for (VMCPUID i = 0; i < pVM->cCpus; i++)
921 {
922 PVMCPU pVCpu = &pVM->aCpus[i];
923 AssertPtr(pVCpu);
924
925 /* Allocate the VM control structure (VMCS). */
926 AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
927 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
928 if (RT_FAILURE(rc))
929 goto cleanup;
930
931 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
932 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
933 {
934 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
935 &pVCpu->hm.s.vmx.HCPhysVirtApic);
936 if (RT_FAILURE(rc))
937 goto cleanup;
938 }
939
940 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
941 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
942 {
943 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
944 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
945 if (RT_FAILURE(rc))
946 goto cleanup;
947 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
948 }
949
950#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
951 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
952 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
953 if (RT_FAILURE(rc))
954 goto cleanup;
955
956 /* Allocate the VM-exit MSR-load page for the host MSRs. */
957 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
958 if (RT_FAILURE(rc))
959 goto cleanup;
960#endif
961 }
962
963 return VINF_SUCCESS;
964
965cleanup:
966 hmR0VmxStructsFree(pVM);
967 return rc;
968}
969
970
971/**
972 * Does global VT-x initialization (called during module initialization).
973 *
974 * @returns VBox status code.
975 */
976VMMR0DECL(int) VMXR0GlobalInit(void)
977{
978#ifdef HMVMX_USE_FUNCTION_TABLE
979 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
980# ifdef VBOX_STRICT
981 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
982 Assert(g_apfnVMExitHandlers[i]);
983# endif
984#endif
985 return VINF_SUCCESS;
986}
987
988
989/**
990 * Does global VT-x termination (called during module termination).
991 */
992VMMR0DECL(void) VMXR0GlobalTerm()
993{
994 /* Nothing to do currently. */
995}
996
997
998/**
999 * Sets up and activates VT-x on the current CPU.
1000 *
1001 * @returns VBox status code.
1002 * @param pCpu Pointer to the global CPU info struct.
1003 * @param pVM Pointer to the VM (can be NULL after a host resume
1004 * operation).
1005 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1006 * fEnabledByHost is true).
1007 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1008 * @a fEnabledByHost is true).
1009 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1010 * enable VT-x on the host.
1011 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1012 */
1013VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1014 void *pvMsrs)
1015{
1016 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1017 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1018 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1019
1020 /* Enable VT-x if it's not already enabled by the host. */
1021 if (!fEnabledByHost)
1022 {
1023 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1024 if (RT_FAILURE(rc))
1025 return rc;
1026 }
1027
1028 /*
1029 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1030 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1031 */
1032 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1033 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1034 {
1035 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1036 pCpu->fFlushAsidBeforeUse = false;
1037 }
1038 else
1039 pCpu->fFlushAsidBeforeUse = true;
1040
1041 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1042 ++pCpu->cTlbFlushes;
1043
1044 return VINF_SUCCESS;
1045}
1046
1047
1048/**
1049 * Deactivates VT-x on the current CPU.
1050 *
1051 * @returns VBox status code.
1052 * @param pCpu Pointer to the global CPU info struct.
1053 * @param pvCpuPage Pointer to the VMXON region.
1054 * @param HCPhysCpuPage Physical address of the VMXON region.
1055 *
1056 * @remarks This function should never be called when SUPR0EnableVTx() or
1057 * similar was used to enable VT-x on the host.
1058 */
1059VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1060{
1061 NOREF(pCpu);
1062 NOREF(pvCpuPage);
1063 NOREF(HCPhysCpuPage);
1064
1065 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1066 return hmR0VmxLeaveRootMode();
1067}
1068
1069
1070/**
1071 * Sets the permission bits for the specified MSR in the MSR bitmap.
1072 *
1073 * @param pVCpu Pointer to the VMCPU.
1074 * @param uMSR The MSR value.
1075 * @param enmRead Whether reading this MSR causes a VM-exit.
1076 * @param enmWrite Whether writing this MSR causes a VM-exit.
1077 */
1078static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1079{
1080 int32_t iBit;
1081 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1082
1083 /*
1084 * Layout:
1085 * 0x000 - 0x3ff - Low MSR read bits
1086 * 0x400 - 0x7ff - High MSR read bits
1087 * 0x800 - 0xbff - Low MSR write bits
1088 * 0xc00 - 0xfff - High MSR write bits
1089 */
1090 if (uMsr <= 0x00001FFF)
1091 iBit = uMsr;
1092 else if ( uMsr >= 0xC0000000
1093 && uMsr <= 0xC0001FFF)
1094 {
1095 iBit = (uMsr - 0xC0000000);
1096 pbMsrBitmap += 0x400;
1097 }
1098 else
1099 {
1100 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1101 return;
1102 }
1103
1104 Assert(iBit <= 0x1fff);
1105 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1106 ASMBitSet(pbMsrBitmap, iBit);
1107 else
1108 ASMBitClear(pbMsrBitmap, iBit);
1109
1110 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1111 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1112 else
1113 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1114}
1115
1116
1117/**
1118 * Flushes the TLB using EPT.
1119 *
1120 * @returns VBox status code.
1121 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1122 * enmFlush).
1123 * @param enmFlush Type of flush.
1124 *
1125 * @remarks Caller is responsible for making sure this function is called only
1126 * when NestedPaging is supported and providing @a enmFlush that is
1127 * supported by the CPU.
1128 */
1129static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1130{
1131 uint64_t au64Descriptor[2];
1132 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1133 au64Descriptor[0] = 0;
1134 else
1135 {
1136 Assert(pVCpu);
1137 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1138 }
1139 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1140
1141 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1142 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1143 rc));
1144 if ( RT_SUCCESS(rc)
1145 && pVCpu)
1146 {
1147 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1148 }
1149}
1150
1151
1152/**
1153 * Flushes the TLB using VPID.
1154 *
1155 * @returns VBox status code.
1156 * @param pVM Pointer to the VM.
1157 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1158 * enmFlush).
1159 * @param enmFlush Type of flush.
1160 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1161 * on @a enmFlush).
1162 */
1163static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1164{
1165 AssertPtr(pVM);
1166 Assert(pVM->hm.s.vmx.fVpid);
1167
1168 uint64_t au64Descriptor[2];
1169 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1170 {
1171 au64Descriptor[0] = 0;
1172 au64Descriptor[1] = 0;
1173 }
1174 else
1175 {
1176 AssertPtr(pVCpu);
1177 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1178 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1179 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1180 au64Descriptor[1] = GCPtr;
1181 }
1182
1183 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1184 AssertMsg(rc == VINF_SUCCESS,
1185 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1186 if ( RT_SUCCESS(rc)
1187 && pVCpu)
1188 {
1189 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1190 }
1191}
1192
1193
1194/**
1195 * Invalidates a guest page by guest virtual address. Only relevant for
1196 * EPT/VPID, otherwise there is nothing really to invalidate.
1197 *
1198 * @returns VBox status code.
1199 * @param pVM Pointer to the VM.
1200 * @param pVCpu Pointer to the VMCPU.
1201 * @param GCVirt Guest virtual address of the page to invalidate.
1202 */
1203VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1204{
1205 AssertPtr(pVM);
1206 AssertPtr(pVCpu);
1207 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1208
1209 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1210 if (!fFlushPending)
1211 {
1212 /*
1213 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1214 * See @bugref{6043} and @bugref{6177}.
1215 *
1216 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1217 * function maybe called in a loop with individual addresses.
1218 */
1219 if (pVM->hm.s.vmx.fVpid)
1220 {
1221 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1222 {
1223 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1224 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1225 }
1226 else
1227 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1228 }
1229 else if (pVM->hm.s.fNestedPaging)
1230 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1231 }
1232
1233 return VINF_SUCCESS;
1234}
1235
1236
1237/**
1238 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1239 * otherwise there is nothing really to invalidate.
1240 *
1241 * @returns VBox status code.
1242 * @param pVM Pointer to the VM.
1243 * @param pVCpu Pointer to the VMCPU.
1244 * @param GCPhys Guest physical address of the page to invalidate.
1245 */
1246VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1247{
1248 LogFlowFunc(("%RGp\n", GCPhys));
1249
1250 /*
1251 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1252 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1253 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1254 */
1255 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1256 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1257 return VINF_SUCCESS;
1258}
1259
1260
1261/**
1262 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1263 * case where neither EPT nor VPID is supported by the CPU.
1264 *
1265 * @param pVM Pointer to the VM.
1266 * @param pVCpu Pointer to the VMCPU.
1267 * @param pCpu Pointer to the global HM struct.
1268 *
1269 * @remarks Called with interrupts disabled.
1270 */
1271static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1272{
1273 AssertPtr(pVCpu);
1274 AssertPtr(pCpu);
1275 NOREF(pVM);
1276
1277 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1278 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1279
1280 pVCpu->hm.s.TlbShootdown.cPages = 0;
1281 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1282 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1283 pVCpu->hm.s.fForceTLBFlush = false;
1284 return;
1285}
1286
1287
1288/**
1289 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1290 *
1291 * @param pVM Pointer to the VM.
1292 * @param pVCpu Pointer to the VMCPU.
1293 * @param pCpu Pointer to the global HM CPU struct.
1294 * @remarks All references to "ASID" in this function pertains to "VPID" in
1295 * Intel's nomenclature. The reason is, to avoid confusion in compare
1296 * statements since the host-CPU copies are named "ASID".
1297 *
1298 * @remarks Called with interrupts disabled.
1299 */
1300static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1301{
1302#ifdef VBOX_WITH_STATISTICS
1303 bool fTlbFlushed = false;
1304# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1305# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1306 if (!fTlbFlushed) \
1307 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1308 } while (0)
1309#else
1310# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1311# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1312#endif
1313
1314 AssertPtr(pVM);
1315 AssertPtr(pCpu);
1316 AssertPtr(pVCpu);
1317 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1318 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1319 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1320
1321
1322 /*
1323 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1324 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1325 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1326 */
1327 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1328 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1329 {
1330 ++pCpu->uCurrentAsid;
1331 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1332 {
1333 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1334 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1335 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1336 }
1337
1338 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1339 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1340 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1341
1342 /*
1343 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1344 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1345 */
1346 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1347 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1348 HMVMX_SET_TAGGED_TLB_FLUSHED();
1349 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1350 }
1351
1352 /* Check for explicit TLB shootdowns. */
1353 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1354 {
1355 /*
1356 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1357 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1358 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1359 * but not guest-physical mappings.
1360 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1361 */
1362 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1363 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1364 HMVMX_SET_TAGGED_TLB_FLUSHED();
1365 }
1366
1367 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1368 * not be executed. See hmQueueInvlPage() where it is commented
1369 * out. Support individual entry flushing someday. */
1370 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1371 {
1372 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1373
1374 /*
1375 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1376 * as supported by the CPU.
1377 */
1378 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1379 {
1380 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1381 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1382 }
1383 else
1384 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1385
1386 HMVMX_SET_TAGGED_TLB_FLUSHED();
1387 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1388 }
1389
1390 pVCpu->hm.s.TlbShootdown.cPages = 0;
1391 pVCpu->hm.s.fForceTLBFlush = false;
1392
1393 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1394
1395 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1396 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1397 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1398 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1399 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1400 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1401 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1402 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1403
1404 /* Update VMCS with the VPID. */
1405 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1406 AssertRC(rc);
1407
1408#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1409}
1410
1411
1412/**
1413 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1414 *
1415 * @returns VBox status code.
1416 * @param pVM Pointer to the VM.
1417 * @param pVCpu Pointer to the VMCPU.
1418 * @param pCpu Pointer to the global HM CPU struct.
1419 *
1420 * @remarks Called with interrupts disabled.
1421 */
1422static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1423{
1424 AssertPtr(pVM);
1425 AssertPtr(pVCpu);
1426 AssertPtr(pCpu);
1427 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1428 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1429
1430 /*
1431 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1432 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1433 */
1434 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1435 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1436 {
1437 pVCpu->hm.s.fForceTLBFlush = true;
1438 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1439 }
1440
1441 /* Check for explicit TLB shootdown flushes. */
1442 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1443 {
1444 pVCpu->hm.s.fForceTLBFlush = true;
1445 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1446 }
1447
1448 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1449 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1450
1451 if (pVCpu->hm.s.fForceTLBFlush)
1452 {
1453 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1454 pVCpu->hm.s.fForceTLBFlush = false;
1455 }
1456 else
1457 {
1458 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1459 * not be executed. See hmQueueInvlPage() where it is commented
1460 * out. Support individual entry flushing someday. */
1461 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1462 {
1463 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1464 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1465 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1466 }
1467 else
1468 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1469 }
1470
1471 pVCpu->hm.s.TlbShootdown.cPages = 0;
1472 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1473}
1474
1475
1476/**
1477 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1478 *
1479 * @returns VBox status code.
1480 * @param pVM Pointer to the VM.
1481 * @param pVCpu Pointer to the VMCPU.
1482 * @param pCpu Pointer to the global HM CPU struct.
1483 *
1484 * @remarks Called with interrupts disabled.
1485 */
1486static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1487{
1488 AssertPtr(pVM);
1489 AssertPtr(pVCpu);
1490 AssertPtr(pCpu);
1491 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1492 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1493
1494 /*
1495 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1496 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1497 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1498 */
1499 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1500 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1501 {
1502 pVCpu->hm.s.fForceTLBFlush = true;
1503 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1504 }
1505
1506 /* Check for explicit TLB shootdown flushes. */
1507 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1508 {
1509 /*
1510 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1511 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1512 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1513 */
1514 pVCpu->hm.s.fForceTLBFlush = true;
1515 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1516 }
1517
1518 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1519 if (pVCpu->hm.s.fForceTLBFlush)
1520 {
1521 ++pCpu->uCurrentAsid;
1522 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1523 {
1524 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1525 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1526 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1527 }
1528
1529 pVCpu->hm.s.fForceTLBFlush = false;
1530 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1531 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1532 if (pCpu->fFlushAsidBeforeUse)
1533 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1534 }
1535 else
1536 {
1537 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1538 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1539 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1540 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1541
1542 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1543 * not be executed. See hmQueueInvlPage() where it is commented
1544 * out. Support individual entry flushing someday. */
1545 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1546 {
1547 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1548 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1549 {
1550 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1551 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1552 }
1553 else
1554 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1555 }
1556 else
1557 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1558 }
1559
1560 pVCpu->hm.s.TlbShootdown.cPages = 0;
1561 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1562
1563 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1564 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1565 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1566 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1567 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1568 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1569
1570 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1571 AssertRC(rc);
1572}
1573
1574
1575/**
1576 * Flushes the guest TLB entry based on CPU capabilities.
1577 *
1578 * @param pVCpu Pointer to the VMCPU.
1579 * @param pCpu Pointer to the global HM CPU struct.
1580 */
1581DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1582{
1583 PVM pVM = pVCpu->CTX_SUFF(pVM);
1584 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1585 {
1586 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
1587 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
1588 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
1589 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
1590 default:
1591 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1592 break;
1593 }
1594}
1595
1596
1597/**
1598 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1599 * TLB entries from the host TLB before VM-entry.
1600 *
1601 * @returns VBox status code.
1602 * @param pVM Pointer to the VM.
1603 */
1604static int hmR0VmxSetupTaggedTlb(PVM pVM)
1605{
1606 /*
1607 * Determine optimal flush type for Nested Paging.
1608 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1609 * guest execution (see hmR3InitFinalizeR0()).
1610 */
1611 if (pVM->hm.s.fNestedPaging)
1612 {
1613 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1614 {
1615 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1616 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1617 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1618 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1619 else
1620 {
1621 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1622 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1623 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1624 }
1625
1626 /* Make sure the write-back cacheable memory type for EPT is supported. */
1627 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1628 {
1629 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1630 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1631 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1632 }
1633 }
1634 else
1635 {
1636 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1637 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1638 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1639 }
1640 }
1641
1642 /*
1643 * Determine optimal flush type for VPID.
1644 */
1645 if (pVM->hm.s.vmx.fVpid)
1646 {
1647 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1648 {
1649 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1650 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1651 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1652 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1653 else
1654 {
1655 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1656 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1657 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1658 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1659 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1660 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1661 pVM->hm.s.vmx.fVpid = false;
1662 }
1663 }
1664 else
1665 {
1666 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1667 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1668 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1669 pVM->hm.s.vmx.fVpid = false;
1670 }
1671 }
1672
1673 /*
1674 * Setup the handler for flushing tagged-TLBs.
1675 */
1676 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1677 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1678 else if (pVM->hm.s.fNestedPaging)
1679 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1680 else if (pVM->hm.s.vmx.fVpid)
1681 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1682 else
1683 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1684 return VINF_SUCCESS;
1685}
1686
1687
1688/**
1689 * Sets up pin-based VM-execution controls in the VMCS.
1690 *
1691 * @returns VBox status code.
1692 * @param pVM Pointer to the VM.
1693 * @param pVCpu Pointer to the VMCPU.
1694 */
1695static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1696{
1697 AssertPtr(pVM);
1698 AssertPtr(pVCpu);
1699
1700 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1701 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1702
1703 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1704 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1705 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1706
1707 /* Enable the VMX preemption timer. */
1708 if (pVM->hm.s.vmx.fUsePreemptTimer)
1709 {
1710 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1711 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1712 }
1713
1714 if ((val & zap) != val)
1715 {
1716 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1717 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1718 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1719 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1720 }
1721
1722 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1723 AssertRCReturn(rc, rc);
1724
1725 /* Update VCPU with the currently set pin-based VM-execution controls. */
1726 pVCpu->hm.s.vmx.u32PinCtls = val;
1727 return rc;
1728}
1729
1730
1731/**
1732 * Sets up processor-based VM-execution controls in the VMCS.
1733 *
1734 * @returns VBox status code.
1735 * @param pVM Pointer to the VM.
1736 * @param pVMCPU Pointer to the VMCPU.
1737 */
1738static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1739{
1740 AssertPtr(pVM);
1741 AssertPtr(pVCpu);
1742
1743 int rc = VERR_INTERNAL_ERROR_5;
1744 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1745 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1746
1747 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1748 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1749 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1750 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1751 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1752 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1753 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1754
1755 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1756 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1757 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1758 {
1759 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1760 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1761 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1762 }
1763
1764 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1765 if (!pVM->hm.s.fNestedPaging)
1766 {
1767 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1768 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1769 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1770 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1771 }
1772
1773 /* Use TPR shadowing if supported by the CPU. */
1774 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1775 {
1776 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1777 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1778 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1779 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1780 AssertRCReturn(rc, rc);
1781
1782 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1783 /* CR8 writes causes a VM-exit based on TPR threshold. */
1784 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1785 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1786 }
1787 else
1788 {
1789 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1790 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1791 }
1792
1793 /* Use MSR-bitmaps if supported by the CPU. */
1794 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1795 {
1796 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1797
1798 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1799 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1800 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1801 AssertRCReturn(rc, rc);
1802
1803 /*
1804 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1805 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1806 */
1807 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1808 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1809 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1810 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1811 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1812 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1813 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1814 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1815 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1816 }
1817
1818 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1819 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1820 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1821
1822 if ((val & zap) != val)
1823 {
1824 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1825 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1826 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1827 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1828 }
1829
1830 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1831 AssertRCReturn(rc, rc);
1832
1833 /* Update VCPU with the currently set processor-based VM-execution controls. */
1834 pVCpu->hm.s.vmx.u32ProcCtls = val;
1835
1836 /*
1837 * Secondary processor-based VM-execution controls.
1838 */
1839 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1840 {
1841 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1842 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1843
1844 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1845 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1846
1847 if (pVM->hm.s.fNestedPaging)
1848 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1849 else
1850 {
1851 /*
1852 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1853 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1854 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1855 */
1856 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1857 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1858 }
1859
1860 if (pVM->hm.s.vmx.fVpid)
1861 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1862
1863 if (pVM->hm.s.vmx.fUnrestrictedGuest)
1864 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1865
1866 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1867 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1868 * done dynamically. */
1869 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1870 {
1871 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1872 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1873 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1874 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1875 AssertRCReturn(rc, rc);
1876 }
1877
1878 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1879 {
1880 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1881 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1882 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1883 }
1884
1885 if ((val & zap) != val)
1886 {
1887 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1888 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1889 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1890 }
1891
1892 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1893 AssertRCReturn(rc, rc);
1894
1895 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1896 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1897 }
1898 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1899 {
1900 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1901 "available\n"));
1902 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1903 }
1904
1905 return VINF_SUCCESS;
1906}
1907
1908
1909/**
1910 * Sets up miscellaneous (everything other than Pin & Processor-based
1911 * VM-execution) control fields in the VMCS.
1912 *
1913 * @returns VBox status code.
1914 * @param pVM Pointer to the VM.
1915 * @param pVCpu Pointer to the VMCPU.
1916 */
1917static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1918{
1919 AssertPtr(pVM);
1920 AssertPtr(pVCpu);
1921
1922 int rc = VERR_GENERAL_FAILURE;
1923
1924 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1925#if 0
1926 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1927 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1928 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1929
1930 /*
1931 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1932 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1933 * We thus use the exception bitmap to control it rather than use both.
1934 */
1935 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1936 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1937
1938 /** @todo Explore possibility of using IO-bitmaps. */
1939 /* All IO & IOIO instructions cause VM-exits. */
1940 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1941 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1942
1943 /* Initialize the MSR-bitmap area. */
1944 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1945 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1946 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1947#endif
1948
1949#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1950 /* Setup MSR autoloading/storing. */
1951 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1952 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1953 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1954 AssertRCReturn(rc, rc);
1955 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1956 AssertRCReturn(rc, rc);
1957
1958 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1959 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1960 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1961 AssertRCReturn(rc, rc);
1962#endif
1963
1964 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1965 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1966 AssertRCReturn(rc, rc);
1967
1968 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1969#if 0
1970 /* Setup debug controls */
1971 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1972 AssertRCReturn(rc, rc);
1973 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1974 AssertRCReturn(rc, rc);
1975#endif
1976
1977 return rc;
1978}
1979
1980
1981/**
1982 * Sets up the initial exception bitmap in the VMCS based on static conditions
1983 * (i.e. conditions that cannot ever change after starting the VM).
1984 *
1985 * @returns VBox status code.
1986 * @param pVM Pointer to the VM.
1987 * @param pVCpu Pointer to the VMCPU.
1988 */
1989static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1990{
1991 AssertPtr(pVM);
1992 AssertPtr(pVCpu);
1993
1994 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1995
1996 uint32_t u32XcptBitmap = 0;
1997
1998 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1999 if (!pVM->hm.s.fNestedPaging)
2000 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2001
2002 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2003 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2004 AssertRCReturn(rc, rc);
2005 return rc;
2006}
2007
2008
2009/**
2010 * Sets up the initial guest-state mask. The guest-state mask is consulted
2011 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2012 * for the nested virtualization case (as it would cause a VM-exit).
2013 *
2014 * @param pVCpu Pointer to the VMCPU.
2015 */
2016static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2017{
2018 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2019 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2020 return VINF_SUCCESS;
2021}
2022
2023
2024/**
2025 * Does per-VM VT-x initialization.
2026 *
2027 * @returns VBox status code.
2028 * @param pVM Pointer to the VM.
2029 */
2030VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2031{
2032 LogFlowFunc(("pVM=%p\n", pVM));
2033
2034 int rc = hmR0VmxStructsAlloc(pVM);
2035 if (RT_FAILURE(rc))
2036 {
2037 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2038 return rc;
2039 }
2040
2041 return VINF_SUCCESS;
2042}
2043
2044
2045/**
2046 * Does per-VM VT-x termination.
2047 *
2048 * @returns VBox status code.
2049 * @param pVM Pointer to the VM.
2050 */
2051VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2052{
2053 LogFlowFunc(("pVM=%p\n", pVM));
2054
2055#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2056 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2057 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2058#endif
2059 hmR0VmxStructsFree(pVM);
2060 return VINF_SUCCESS;
2061}
2062
2063
2064/**
2065 * Sets up the VM for execution under VT-x.
2066 * This function is only called once per-VM during initialization.
2067 *
2068 * @returns VBox status code.
2069 * @param pVM Pointer to the VM.
2070 */
2071VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2072{
2073 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2074 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2075
2076 LogFlowFunc(("pVM=%p\n", pVM));
2077
2078 /*
2079 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2080 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2081 */
2082 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2083 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2084 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2085 || !pVM->hm.s.vmx.pRealModeTSS))
2086 {
2087 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2088 return VERR_INTERNAL_ERROR;
2089 }
2090
2091#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2092 /*
2093 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2094 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2095 */
2096 if ( pVM->hm.s.fAllow64BitGuests
2097 && !HMVMX_IS_64BIT_HOST_MODE())
2098 {
2099 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2100 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2101 }
2102#endif
2103
2104 /* Initialize these always, see hmR3InitFinalizeR0().*/
2105 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2106 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2107
2108 /* Setup the tagged-TLB flush handlers. */
2109 int rc = hmR0VmxSetupTaggedTlb(pVM);
2110 if (RT_FAILURE(rc))
2111 {
2112 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2113 return rc;
2114 }
2115
2116 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2117 {
2118 PVMCPU pVCpu = &pVM->aCpus[i];
2119 AssertPtr(pVCpu);
2120 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2121
2122 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2123 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2124
2125 /* Set revision dword at the beginning of the VMCS structure. */
2126 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2127
2128 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2129 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2130 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2131 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2132
2133 /* Load this VMCS as the current VMCS. */
2134 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2135 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2136 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2137
2138 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2139 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2140 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2141
2142 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2143 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2144 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2145
2146 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2147 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2148 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2149
2150 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2151 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2152 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2153
2154 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2155 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2156 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2157
2158#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2159 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2160 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2161 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2162#endif
2163
2164 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2165 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2166 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2167 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2168
2169 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2170
2171 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2172 }
2173
2174 return VINF_SUCCESS;
2175}
2176
2177
2178/**
2179 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2180 * the VMCS.
2181 *
2182 * @returns VBox status code.
2183 * @param pVM Pointer to the VM.
2184 * @param pVCpu Pointer to the VMCPU.
2185 */
2186DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2187{
2188 RTCCUINTREG uReg = ASMGetCR0();
2189 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2190 AssertRCReturn(rc, rc);
2191
2192#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2193 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2194 if (HMVMX_IS_64BIT_HOST_MODE())
2195 {
2196 uint64_t uRegCR3 = HMR0Get64bitCR3();
2197 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2198 }
2199 else
2200#endif
2201 {
2202 uReg = ASMGetCR3();
2203 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2204 }
2205 AssertRCReturn(rc, rc);
2206
2207 uReg = ASMGetCR4();
2208 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2209 AssertRCReturn(rc, rc);
2210 return rc;
2211}
2212
2213
2214/**
2215 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2216 * the host-state area in the VMCS.
2217 *
2218 * @returns VBox status code.
2219 * @param pVM Pointer to the VM.
2220 * @param pVCpu Pointer to the VMCPU.
2221 */
2222DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2223{
2224 int rc = VERR_INTERNAL_ERROR_5;
2225 RTSEL uSelDS = 0;
2226 RTSEL uSelES = 0;
2227 RTSEL uSelFS = 0;
2228 RTSEL uSelGS = 0;
2229 RTSEL uSelTR = 0;
2230
2231 /*
2232 * Host DS, ES, FS and GS segment registers.
2233 */
2234#if HC_ARCH_BITS == 64
2235 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2236 uSelDS = ASMGetDS();
2237 uSelES = ASMGetES();
2238 uSelFS = ASMGetFS();
2239 uSelGS = ASMGetGS();
2240#endif
2241
2242 /*
2243 * Host CS and SS segment registers.
2244 */
2245 RTSEL uSelCS;
2246 RTSEL uSelSS;
2247#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2248 if (HMVMX_IS_64BIT_HOST_MODE())
2249 {
2250 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2251 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2252 }
2253 else
2254 {
2255 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2256 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2257 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2258 }
2259#else
2260 uSelCS = ASMGetCS();
2261 uSelSS = ASMGetSS();
2262#endif
2263
2264 /*
2265 * Host TR segment register.
2266 */
2267 uSelTR = ASMGetTR();
2268
2269#if HC_ARCH_BITS == 64
2270 /*
2271 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2272 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2273 */
2274 if (uSelDS & (X86_SEL_RPL | X86_SEL_LDT))
2275 {
2276 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_DS;
2277 pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2278 uSelDS = 0;
2279 }
2280 if (uSelES & (X86_SEL_RPL | X86_SEL_LDT))
2281 {
2282 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_ES;
2283 pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2284 uSelES = 0;
2285 }
2286 if (uSelFS & (X86_SEL_RPL | X86_SEL_LDT))
2287 {
2288 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_FS;
2289 pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2290 uSelFS = 0;
2291 }
2292 if (uSelGS & (X86_SEL_RPL | X86_SEL_LDT))
2293 {
2294 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_GS;
2295 pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2296 uSelGS = 0;
2297 }
2298#endif
2299
2300 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2301 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2302 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2303 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2304 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2305 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2306 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2307 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2308 Assert(uSelCS);
2309 Assert(uSelTR);
2310
2311 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2312#if 0
2313 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2314 Assert(uSelSS != 0);
2315#endif
2316
2317 /* Write these host selector fields into the host-state area in the VMCS. */
2318 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2319 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2320#if HC_ARCH_BITS == 64
2321 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2322 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2323 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2324 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2325#endif
2326 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2327
2328 /*
2329 * Host GDTR and IDTR.
2330 */
2331 RTGDTR Gdtr;
2332 RT_ZERO(Gdtr);
2333#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2334 if (HMVMX_IS_64BIT_HOST_MODE())
2335 {
2336 X86XDTR64 Gdtr64;
2337 X86XDTR64 Idtr64;
2338 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2339 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2340 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2341
2342 Gdtr.cbGdt = Gdtr64.cb;
2343 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2344 }
2345 else
2346#endif
2347 {
2348 RTIDTR Idtr;
2349 ASMGetGDTR(&Gdtr);
2350 ASMGetIDTR(&Idtr);
2351 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2352 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2353
2354#if HC_ARCH_BITS == 64
2355 /*
2356 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2357 * maximum limit (0xffff) on every VM-exit.
2358 */
2359 if (Gdtr.cbGdt != 0xffff)
2360 {
2361 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2362 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2363 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2364 }
2365
2366 /*
2367 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2368 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2369 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2370 */
2371 if (Idtr.cbIdt < 0x0fff)
2372 {
2373 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2374 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2375 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2376 }
2377#endif
2378 }
2379
2380 /*
2381 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2382 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2383 */
2384 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2385 {
2386 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2387 return VERR_VMX_INVALID_HOST_STATE;
2388 }
2389
2390 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2391#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2392 if (HMVMX_IS_64BIT_HOST_MODE())
2393 {
2394 /* We need the 64-bit TR base for hybrid darwin. */
2395 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2396 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2397 }
2398 else
2399#endif
2400 {
2401 uintptr_t uTRBase;
2402#if HC_ARCH_BITS == 64
2403 uTRBase = X86DESC64_BASE(pDesc);
2404
2405 /*
2406 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2407 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2408 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2409 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2410 *
2411 * [1] See Intel spec. 3.5 "System Descriptor Types".
2412 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2413 */
2414 Assert(pDesc->System.u4Type == 11);
2415 if ( pDesc->System.u16LimitLow != 0x67
2416 || pDesc->System.u4LimitHigh)
2417 {
2418 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2419 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2420
2421 /* Store the GDTR here as we need it while restoring TR. */
2422 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2423 }
2424#else
2425 uTRBase = X86DESC_BASE(pDesc);
2426#endif
2427 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2428 }
2429 AssertRCReturn(rc, rc);
2430
2431 /*
2432 * Host FS base and GS base.
2433 */
2434#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2435 if (HMVMX_IS_64BIT_HOST_MODE())
2436 {
2437 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2438 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2439 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2440 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2441
2442# if HC_ARCH_BITS == 64
2443 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2444 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2445 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2446 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2447 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2448# endif
2449 }
2450#endif
2451 return rc;
2452}
2453
2454
2455/**
2456 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2457 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2458 * the host after every successful VM exit.
2459 *
2460 * @returns VBox status code.
2461 * @param pVM Pointer to the VM.
2462 * @param pVCpu Pointer to the VMCPU.
2463 */
2464DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2465{
2466 AssertPtr(pVCpu);
2467 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2468
2469 int rc = VINF_SUCCESS;
2470#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2471 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2472 uint32_t cHostMsrs = 0;
2473 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2474
2475 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2476 {
2477 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2478
2479# if HC_ARCH_BITS == 64
2480 /* Paranoia. 64-bit code requires these bits to be set always. */
2481 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2482
2483 /*
2484 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2485 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2486 * some reason (e.g. allow transparent reads) we would activate the code below.
2487 */
2488# if 0
2489 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2490 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2491 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2492 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2493 if (CPUMIsGuestInLongMode(pVCpu))
2494 {
2495 uint64_t u64GuestEfer;
2496 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2497 AssertRC(rc);
2498
2499 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2500 {
2501 pHostMsr->u32Msr = MSR_K6_EFER;
2502 pHostMsr->u32Reserved = 0;
2503 pHostMsr->u64Value = u64HostEfer;
2504 pHostMsr++; cHostMsrs++;
2505 }
2506 }
2507# endif
2508# else /* HC_ARCH_BITS != 64 */
2509 pHostMsr->u32Msr = MSR_K6_EFER;
2510 pHostMsr->u32Reserved = 0;
2511# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2512 if (CPUMIsGuestInLongMode(pVCpu))
2513 {
2514 /* Must match the EFER value in our 64 bits switcher. */
2515 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2516 }
2517 else
2518# endif
2519 pHostMsr->u64Value = u64HostEfer;
2520 pHostMsr++; cHostMsrs++;
2521# endif /* HC_ARCH_BITS == 64 */
2522 }
2523
2524# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2525 if (HMVMX_IS_64BIT_HOST_MODE())
2526 {
2527 pHostMsr->u32Msr = MSR_K6_STAR;
2528 pHostMsr->u32Reserved = 0;
2529 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2530 pHostMsr++; cHostMsrs++;
2531 pHostMsr->u32Msr = MSR_K8_LSTAR;
2532 pHostMsr->u32Reserved = 0;
2533 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2534 pHostMsr++; cHostMsrs++;
2535 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2536 pHostMsr->u32Reserved = 0;
2537 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2538 pHostMsr++; cHostMsrs++;
2539 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2540 pHostMsr->u32Reserved = 0;
2541 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2542 pHostMsr++; cHostMsrs++;
2543 }
2544# endif
2545
2546 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2547 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2548 {
2549 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2550 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2551 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2552 }
2553
2554 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2555#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2556
2557 /*
2558 * Host Sysenter MSRs.
2559 */
2560 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2561 AssertRCReturn(rc, rc);
2562#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2563 if (HMVMX_IS_64BIT_HOST_MODE())
2564 {
2565 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2566 AssertRCReturn(rc, rc);
2567 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2568 }
2569 else
2570 {
2571 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2572 AssertRCReturn(rc, rc);
2573 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2574 }
2575#elif HC_ARCH_BITS == 32
2576 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2577 AssertRCReturn(rc, rc);
2578 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2579#else
2580 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2581 AssertRCReturn(rc, rc);
2582 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2583#endif
2584 AssertRCReturn(rc, rc);
2585
2586 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2587 * hmR0VmxSetupExitCtls() !! */
2588 return rc;
2589}
2590
2591
2592/**
2593 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2594 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2595 * controls".
2596 *
2597 * @returns VBox status code.
2598 * @param pVCpu Pointer to the VMCPU.
2599 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2600 * out-of-sync. Make sure to update the required fields
2601 * before using them.
2602 *
2603 * @remarks No-long-jump zone!!!
2604 */
2605DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2606{
2607 int rc = VINF_SUCCESS;
2608 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2609 {
2610 PVM pVM = pVCpu->CTX_SUFF(pVM);
2611 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2612 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2613
2614 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2615 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2616
2617 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2618 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2619 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2620 else
2621 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2622
2623 /*
2624 * The following should not be set (since we're not in SMM mode):
2625 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2626 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2627 */
2628
2629 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2630 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2631 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2632
2633 if ((val & zap) != val)
2634 {
2635 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2636 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2637 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2638 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2639 }
2640
2641 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2642 AssertRCReturn(rc, rc);
2643
2644 /* Update VCPU with the currently set VM-exit controls. */
2645 pVCpu->hm.s.vmx.u32EntryCtls = val;
2646 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2647 }
2648 return rc;
2649}
2650
2651
2652/**
2653 * Sets up the VM-exit controls in the VMCS.
2654 *
2655 * @returns VBox status code.
2656 * @param pVM Pointer to the VM.
2657 * @param pVCpu Pointer to the VMCPU.
2658 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2659 * out-of-sync. Make sure to update the required fields
2660 * before using them.
2661 *
2662 * @remarks requires EFER.
2663 */
2664DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2665{
2666 int rc = VINF_SUCCESS;
2667 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2668 {
2669 PVM pVM = pVCpu->CTX_SUFF(pVM);
2670 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2671 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2672
2673 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2674 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2675
2676 /*
2677 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2678 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2679 */
2680#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2681 if (HMVMX_IS_64BIT_HOST_MODE())
2682 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2683 else
2684 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2685#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2686 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2687 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2688 else
2689 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2690#endif
2691
2692 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2693 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2694
2695 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2696 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2697 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2698 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2699 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2700
2701 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2702 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2703
2704 if ((val & zap) != val)
2705 {
2706 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2707 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2708 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2709 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2710 }
2711
2712 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2713 AssertRCReturn(rc, rc);
2714
2715 /* Update VCPU with the currently set VM-exit controls. */
2716 pVCpu->hm.s.vmx.u32ExitCtls = val;
2717 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2718 }
2719 return rc;
2720}
2721
2722
2723/**
2724 * Loads the guest APIC and related state.
2725 *
2726 * @returns VBox status code.
2727 * @param pVM Pointer to the VM.
2728 * @param pVCpu Pointer to the VMCPU.
2729 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2730 * out-of-sync. Make sure to update the required fields
2731 * before using them.
2732 */
2733DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2734{
2735 int rc = VINF_SUCCESS;
2736 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2737 {
2738 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2739 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2740 {
2741 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2742
2743 bool fPendingIntr = false;
2744 uint8_t u8Tpr = 0;
2745 uint8_t u8PendingIntr = 0;
2746 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2747 AssertRCReturn(rc, rc);
2748
2749 /*
2750 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2751 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2752 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2753 * the interrupt when we VM-exit for other reasons.
2754 */
2755 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2756 uint32_t u32TprThreshold = 0;
2757 if (fPendingIntr)
2758 {
2759 /* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2760 const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2761 const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2762 if (u8PendingPriority <= u8TprPriority)
2763 u32TprThreshold = u8PendingPriority;
2764 else
2765 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2766 }
2767 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2768
2769 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2770 AssertRCReturn(rc, rc);
2771 }
2772
2773 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2774 }
2775 return rc;
2776}
2777
2778
2779/**
2780 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2781 *
2782 * @returns Guest's interruptibility-state.
2783 * @param pVCpu Pointer to the VMCPU.
2784 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2785 * out-of-sync. Make sure to update the required fields
2786 * before using them.
2787 *
2788 * @remarks No-long-jump zone!!!
2789 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2790 */
2791DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2792{
2793 /*
2794 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2795 * inhibit interrupts or clear any existing interrupt-inhibition.
2796 */
2797 uint32_t uIntrState = 0;
2798 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2799 {
2800 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2801 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
2802 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2803 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2804 {
2805 /*
2806 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2807 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2808 */
2809 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2810 }
2811 else if (pMixedCtx->eflags.Bits.u1IF)
2812 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2813 else
2814 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2815 }
2816 return uIntrState;
2817}
2818
2819
2820/**
2821 * Loads the guest's interruptibility-state into the guest-state area in the
2822 * VMCS.
2823 *
2824 * @returns VBox status code.
2825 * @param pVCpu Pointer to the VMCPU.
2826 * @param uIntrState The interruptibility-state to set.
2827 */
2828static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2829{
2830 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2831 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2832 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2833 AssertRCReturn(rc, rc);
2834 return rc;
2835}
2836
2837
2838/**
2839 * Loads the guest's RIP into the guest-state area in the VMCS.
2840 *
2841 * @returns VBox status code.
2842 * @param pVCpu Pointer to the VMCPU.
2843 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2844 * out-of-sync. Make sure to update the required fields
2845 * before using them.
2846 *
2847 * @remarks No-long-jump zone!!!
2848 */
2849static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2850{
2851 int rc = VINF_SUCCESS;
2852 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2853 {
2854 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2855 AssertRCReturn(rc, rc);
2856 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2857 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#x\n", pMixedCtx->rip, pVCpu->hm.s.fContextUseFlags));
2858 }
2859 return rc;
2860}
2861
2862
2863/**
2864 * Loads the guest's RSP into the guest-state area in the VMCS.
2865 *
2866 * @returns VBox status code.
2867 * @param pVCpu Pointer to the VMCPU.
2868 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2869 * out-of-sync. Make sure to update the required fields
2870 * before using them.
2871 *
2872 * @remarks No-long-jump zone!!!
2873 */
2874static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2875{
2876 int rc = VINF_SUCCESS;
2877 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2878 {
2879 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2880 AssertRCReturn(rc, rc);
2881 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2882 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2883 }
2884 return rc;
2885}
2886
2887
2888/**
2889 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
2890 *
2891 * @returns VBox status code.
2892 * @param pVCpu Pointer to the VMCPU.
2893 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2894 * out-of-sync. Make sure to update the required fields
2895 * before using them.
2896 *
2897 * @remarks No-long-jump zone!!!
2898 */
2899static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2900{
2901 int rc = VINF_SUCCESS;
2902 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2903 {
2904 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2905 Let us assert it as such and use 32-bit VMWRITE. */
2906 Assert(!(pMixedCtx->rflags.u64 >> 32));
2907 X86EFLAGS Eflags = pMixedCtx->eflags;
2908 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2909 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2910
2911 /*
2912 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2913 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2914 */
2915 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2916 {
2917 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2918 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2919 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2920 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2921 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2922 }
2923
2924 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2925 AssertRCReturn(rc, rc);
2926
2927 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2928 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2929 }
2930 return rc;
2931}
2932
2933
2934/**
2935 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2936 *
2937 * @returns VBox status code.
2938 * @param pVCpu Pointer to the VMCPU.
2939 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2940 * out-of-sync. Make sure to update the required fields
2941 * before using them.
2942 *
2943 * @remarks No-long-jump zone!!!
2944 */
2945DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2946{
2947 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2948 AssertRCReturn(rc, rc);
2949 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2950 AssertRCReturn(rc, rc);
2951 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2952 AssertRCReturn(rc, rc);
2953 return rc;
2954}
2955
2956
2957/**
2958 * Loads the guest CR0 control register into the guest-state area in the VMCS.
2959 * CR0 is partially shared with the host and we have to consider the FPU bits.
2960 *
2961 * @returns VBox status code.
2962 * @param pVM Pointer to the VM.
2963 * @param pVCpu Pointer to the VMCPU.
2964 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2965 * out-of-sync. Make sure to update the required fields
2966 * before using them.
2967 *
2968 * @remarks No-long-jump zone!!!
2969 */
2970static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2971{
2972 /*
2973 * Guest CR0.
2974 * Guest FPU.
2975 */
2976 int rc = VINF_SUCCESS;
2977 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2978 {
2979 Assert(!(pMixedCtx->cr0 >> 32));
2980 uint32_t u32GuestCR0 = pMixedCtx->cr0;
2981 PVM pVM = pVCpu->CTX_SUFF(pVM);
2982
2983 /* The guest's view (read access) of its CR0 is unblemished. */
2984 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2985 AssertRCReturn(rc, rc);
2986 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2987
2988 /* Setup VT-x's view of the guest CR0. */
2989 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2990 if (pVM->hm.s.fNestedPaging)
2991 {
2992 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
2993 {
2994 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2995 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2996 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2997 }
2998 else
2999 {
3000 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3001 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3002 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3003 }
3004
3005 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3006 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3007 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3008
3009 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3010 AssertRCReturn(rc, rc);
3011 }
3012 else
3013 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3014
3015 /*
3016 * Guest FPU bits.
3017 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3018 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3019 */
3020 u32GuestCR0 |= X86_CR0_NE;
3021 bool fInterceptNM = false;
3022 if (CPUMIsGuestFPUStateActive(pVCpu))
3023 {
3024 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3025 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3026 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3027 }
3028 else
3029 {
3030 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3031 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3032 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3033 }
3034
3035 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3036 bool fInterceptMF = false;
3037 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3038 fInterceptMF = true;
3039
3040 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3041 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3042 {
3043 Assert(PDMVmmDevHeapIsEnabled(pVM));
3044 Assert(pVM->hm.s.vmx.pRealModeTSS);
3045 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3046 fInterceptNM = true;
3047 fInterceptMF = true;
3048 }
3049 else
3050 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3051
3052 if (fInterceptNM)
3053 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3054 else
3055 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3056
3057 if (fInterceptMF)
3058 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3059 else
3060 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3061
3062 /* Additional intercepts for debugging, define these yourself explicitly. */
3063#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3064 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3065 | RT_BIT(X86_XCPT_BP)
3066 | RT_BIT(X86_XCPT_DB)
3067 | RT_BIT(X86_XCPT_DE)
3068 | RT_BIT(X86_XCPT_NM)
3069 | RT_BIT(X86_XCPT_UD)
3070 | RT_BIT(X86_XCPT_NP)
3071 | RT_BIT(X86_XCPT_SS)
3072 | RT_BIT(X86_XCPT_GP)
3073 | RT_BIT(X86_XCPT_PF)
3074 | RT_BIT(X86_XCPT_MF)
3075 ;
3076#elif defined(HMVMX_ALWAYS_TRAP_PF)
3077 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3078#endif
3079
3080 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3081
3082 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3083 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3084 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3085 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3086 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3087 else
3088 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3089
3090 u32GuestCR0 |= uSetCR0;
3091 u32GuestCR0 &= uZapCR0;
3092 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3093
3094 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3095 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3096 AssertRCReturn(rc, rc);
3097 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3098 AssertRCReturn(rc, rc);
3099 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3100
3101 /*
3102 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3103 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3104 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3105 */
3106 uint32_t u32CR0Mask = 0;
3107 u32CR0Mask = X86_CR0_PE
3108 | X86_CR0_NE
3109 | X86_CR0_WP
3110 | X86_CR0_PG
3111 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3112 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3113 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3114 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3115 u32CR0Mask &= ~X86_CR0_PE;
3116 if (pVM->hm.s.fNestedPaging)
3117 u32CR0Mask &= ~X86_CR0_WP;
3118
3119 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3120 if (fInterceptNM)
3121 u32CR0Mask |= (X86_CR0_TS | X86_CR0_MP);
3122 else
3123 u32CR0Mask &= ~(X86_CR0_TS | X86_CR0_MP);
3124
3125 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3126 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3127 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3128 AssertRCReturn(rc, rc);
3129
3130 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3131 }
3132 return rc;
3133}
3134
3135
3136/**
3137 * Loads the guest control registers (CR3, CR4) into the guest-state area
3138 * in the VMCS.
3139 *
3140 * @returns VBox status code.
3141 * @param pVM Pointer to the VM.
3142 * @param pVCpu Pointer to the VMCPU.
3143 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3144 * out-of-sync. Make sure to update the required fields
3145 * before using them.
3146 *
3147 * @remarks No-long-jump zone!!!
3148 */
3149static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3150{
3151 int rc = VINF_SUCCESS;
3152 PVM pVM = pVCpu->CTX_SUFF(pVM);
3153
3154 /*
3155 * Guest CR2.
3156 * It's always loaded in the assembler code. Nothing to do here.
3157 */
3158
3159 /*
3160 * Guest CR3.
3161 */
3162 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3163 {
3164 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3165 if (pVM->hm.s.fNestedPaging)
3166 {
3167 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3168
3169 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3170 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3171 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3172 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3173
3174 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3175 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3176 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3177
3178 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3179 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3180 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3181 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3182
3183 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3184 AssertRCReturn(rc, rc);
3185 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3186
3187 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3188 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3189 {
3190 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3191 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3192 {
3193 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3194 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3195 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3196 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3197 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3198 }
3199
3200 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3201 have Unrestricted Execution to handle the guest when it's not using paging. */
3202 GCPhysGuestCR3 = pMixedCtx->cr3;
3203 }
3204 else
3205 {
3206 /*
3207 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3208 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3209 * EPT takes care of translating it to host-physical addresses.
3210 */
3211 RTGCPHYS GCPhys;
3212 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3213 Assert(PDMVmmDevHeapIsEnabled(pVM));
3214
3215 /* We obtain it here every time as the guest could have relocated this PCI region. */
3216 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3217 AssertRCReturn(rc, rc);
3218
3219 GCPhysGuestCR3 = GCPhys;
3220 }
3221
3222 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3223 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3224 }
3225 else
3226 {
3227 /* Non-nested paging case, just use the hypervisor's CR3. */
3228 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3229
3230 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3231 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3232 }
3233 AssertRCReturn(rc, rc);
3234
3235 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3236 }
3237
3238 /*
3239 * Guest CR4.
3240 */
3241 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3242 {
3243 Assert(!(pMixedCtx->cr4 >> 32));
3244 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3245
3246 /* The guest's view of its CR4 is unblemished. */
3247 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3248 AssertRCReturn(rc, rc);
3249 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3250
3251 /* Setup VT-x's view of the guest CR4. */
3252 /*
3253 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3254 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3255 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3256 */
3257 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3258 {
3259 Assert(pVM->hm.s.vmx.pRealModeTSS);
3260 Assert(PDMVmmDevHeapIsEnabled(pVM));
3261 u32GuestCR4 &= ~X86_CR4_VME;
3262 }
3263
3264 if (pVM->hm.s.fNestedPaging)
3265 {
3266 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3267 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3268 {
3269 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3270 u32GuestCR4 |= X86_CR4_PSE;
3271 /* Our identity mapping is a 32 bits page directory. */
3272 u32GuestCR4 &= ~X86_CR4_PAE;
3273 }
3274 /* else use guest CR4.*/
3275 }
3276 else
3277 {
3278 /*
3279 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3280 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3281 */
3282 switch (pVCpu->hm.s.enmShadowMode)
3283 {
3284 case PGMMODE_REAL: /* Real-mode. */
3285 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3286 case PGMMODE_32_BIT: /* 32-bit paging. */
3287 {
3288 u32GuestCR4 &= ~X86_CR4_PAE;
3289 break;
3290 }
3291
3292 case PGMMODE_PAE: /* PAE paging. */
3293 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3294 {
3295 u32GuestCR4 |= X86_CR4_PAE;
3296 break;
3297 }
3298
3299 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3300 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3301#ifdef VBOX_ENABLE_64_BITS_GUESTS
3302 break;
3303#endif
3304 default:
3305 AssertFailed();
3306 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3307 }
3308 }
3309
3310 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3311 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3312 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3313 u32GuestCR4 |= uSetCR4;
3314 u32GuestCR4 &= uZapCR4;
3315
3316 /* Write VT-x's view of the guest CR4 into the VMCS. */
3317 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3318 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3319 AssertRCReturn(rc, rc);
3320
3321 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3322 uint32_t u32CR4Mask = 0;
3323 u32CR4Mask = X86_CR4_VME
3324 | X86_CR4_PAE
3325 | X86_CR4_PGE
3326 | X86_CR4_PSE
3327 | X86_CR4_VMXE;
3328 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3329 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3330 AssertRCReturn(rc, rc);
3331
3332 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3333 }
3334 return rc;
3335}
3336
3337
3338/**
3339 * Loads the guest debug registers into the guest-state area in the VMCS.
3340 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3341 *
3342 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3343 *
3344 * @returns VBox status code.
3345 * @param pVCpu Pointer to the VMCPU.
3346 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3347 * out-of-sync. Make sure to update the required fields
3348 * before using them.
3349 *
3350 * @remarks No-long-jump zone!!!
3351 */
3352static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3353{
3354 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3355 return VINF_SUCCESS;
3356
3357#ifdef VBOX_STRICT
3358 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3359 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3360 {
3361 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3362 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3363 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3364 }
3365#endif
3366
3367 int rc;
3368 PVM pVM = pVCpu->CTX_SUFF(pVM);
3369 bool fInterceptDB = false;
3370 bool fInterceptMovDRx = false;
3371 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3372 {
3373 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3374 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3375 {
3376 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3377 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3378 AssertRCReturn(rc, rc);
3379 Assert(fInterceptDB == false);
3380 }
3381 else
3382 {
3383 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3384 pVCpu->hm.s.fClearTrapFlag = true;
3385 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RFLAGS;
3386 fInterceptDB = true;
3387 }
3388 }
3389
3390 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3391 {
3392 /*
3393 * Use the combined guest and host DRx values found in the hypervisor
3394 * register set because the debugger has breakpoints active or someone
3395 * is single stepping on the host side without a monitor trap flag.
3396 *
3397 * Note! DBGF expects a clean DR6 state before executing guest code.
3398 */
3399 if (!CPUMIsHyperDebugStateActive(pVCpu))
3400 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3401 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3402 Assert(CPUMIsHyperDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3403
3404 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3405 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3406 AssertRCReturn(rc, rc);
3407
3408 fInterceptDB = true;
3409 fInterceptMovDRx = true;
3410 }
3411 else
3412 {
3413 /*
3414 * If the guest has enabled debug registers, we need to load them prior to
3415 * executing guest code so they'll trigger at the right time.
3416 */
3417 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3418 {
3419 if (!CPUMIsGuestDebugStateActive(pVCpu))
3420 {
3421 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3422 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3423 }
3424 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3425 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3426 }
3427 /*
3428 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3429 * must intercept #DB in order to maintain a correct DR6 guest value.
3430 */
3431 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3432 {
3433 fInterceptMovDRx = true;
3434 fInterceptDB = true;
3435 }
3436
3437 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3438 AssertRCReturn(rc, rc);
3439 }
3440
3441 /*
3442 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3443 */
3444 if (fInterceptDB)
3445 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3446 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3447 {
3448#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3449 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3450#endif
3451 }
3452 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3453 AssertRCReturn(rc, rc);
3454
3455 /*
3456 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3457 */
3458 if (fInterceptMovDRx)
3459 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3460 else
3461 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3462 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3463 AssertRCReturn(rc, rc);
3464
3465 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3466 return VINF_SUCCESS;
3467}
3468
3469
3470#ifdef VBOX_STRICT
3471/**
3472 * Strict function to validate segment registers.
3473 *
3474 * @remarks ASSUMES CR0 is up to date.
3475 */
3476static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3477{
3478 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3479 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3480 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3481 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3482 && ( !CPUMIsGuestInRealModeEx(pCtx)
3483 && !CPUMIsGuestInV86ModeEx(pCtx)))
3484 {
3485 /* Protected mode checks */
3486 /* CS */
3487 Assert(pCtx->cs.Attr.n.u1Present);
3488 Assert(!(pCtx->cs.Attr.u & 0xf00));
3489 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3490 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3491 || !(pCtx->cs.Attr.n.u1Granularity));
3492 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3493 || (pCtx->cs.Attr.n.u1Granularity));
3494 /* CS cannot be loaded with NULL in protected mode. */
3495 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3496 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3497 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3498 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3499 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3500 else
3501 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3502 /* SS */
3503 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3504 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3505 if ( !(pCtx->cr0 & X86_CR0_PE)
3506 || pCtx->cs.Attr.n.u4Type == 3)
3507 {
3508 Assert(!pCtx->ss.Attr.n.u2Dpl);
3509 }
3510 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3511 {
3512 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3513 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3514 Assert(pCtx->ss.Attr.n.u1Present);
3515 Assert(!(pCtx->ss.Attr.u & 0xf00));
3516 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3517 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3518 || !(pCtx->ss.Attr.n.u1Granularity));
3519 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3520 || (pCtx->ss.Attr.n.u1Granularity));
3521 }
3522 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3523 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3524 {
3525 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3526 Assert(pCtx->ds.Attr.n.u1Present);
3527 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3528 Assert(!(pCtx->ds.Attr.u & 0xf00));
3529 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3530 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3531 || !(pCtx->ds.Attr.n.u1Granularity));
3532 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3533 || (pCtx->ds.Attr.n.u1Granularity));
3534 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3535 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3536 }
3537 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3538 {
3539 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3540 Assert(pCtx->es.Attr.n.u1Present);
3541 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3542 Assert(!(pCtx->es.Attr.u & 0xf00));
3543 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3544 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3545 || !(pCtx->es.Attr.n.u1Granularity));
3546 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3547 || (pCtx->es.Attr.n.u1Granularity));
3548 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3549 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3550 }
3551 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3552 {
3553 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3554 Assert(pCtx->fs.Attr.n.u1Present);
3555 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3556 Assert(!(pCtx->fs.Attr.u & 0xf00));
3557 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3558 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3559 || !(pCtx->fs.Attr.n.u1Granularity));
3560 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3561 || (pCtx->fs.Attr.n.u1Granularity));
3562 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3563 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3564 }
3565 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3566 {
3567 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3568 Assert(pCtx->gs.Attr.n.u1Present);
3569 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3570 Assert(!(pCtx->gs.Attr.u & 0xf00));
3571 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3572 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3573 || !(pCtx->gs.Attr.n.u1Granularity));
3574 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3575 || (pCtx->gs.Attr.n.u1Granularity));
3576 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3577 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3578 }
3579 /* 64-bit capable CPUs. */
3580# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3581 Assert(!(pCtx->cs.u64Base >> 32));
3582 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3583 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3584 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3585# endif
3586 }
3587 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3588 || ( CPUMIsGuestInRealModeEx(pCtx)
3589 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3590 {
3591 /* Real and v86 mode checks. */
3592 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3593 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3594 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3595 {
3596 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3597 }
3598 else
3599 {
3600 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3601 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3602 }
3603
3604 /* CS */
3605 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3606 Assert(pCtx->cs.u32Limit == 0xffff);
3607 Assert(u32CSAttr == 0xf3);
3608 /* SS */
3609 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3610 Assert(pCtx->ss.u32Limit == 0xffff);
3611 Assert(u32SSAttr == 0xf3);
3612 /* DS */
3613 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3614 Assert(pCtx->ds.u32Limit == 0xffff);
3615 Assert(u32DSAttr == 0xf3);
3616 /* ES */
3617 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3618 Assert(pCtx->es.u32Limit == 0xffff);
3619 Assert(u32ESAttr == 0xf3);
3620 /* FS */
3621 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3622 Assert(pCtx->fs.u32Limit == 0xffff);
3623 Assert(u32FSAttr == 0xf3);
3624 /* GS */
3625 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3626 Assert(pCtx->gs.u32Limit == 0xffff);
3627 Assert(u32GSAttr == 0xf3);
3628 /* 64-bit capable CPUs. */
3629# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3630 Assert(!(pCtx->cs.u64Base >> 32));
3631 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3632 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3633 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3634# endif
3635 }
3636}
3637#endif /* VBOX_STRICT */
3638
3639
3640/**
3641 * Writes a guest segment register into the guest-state area in the VMCS.
3642 *
3643 * @returns VBox status code.
3644 * @param pVCpu Pointer to the VMCPU.
3645 * @param idxSel Index of the selector in the VMCS.
3646 * @param idxLimit Index of the segment limit in the VMCS.
3647 * @param idxBase Index of the segment base in the VMCS.
3648 * @param idxAccess Index of the access rights of the segment in the VMCS.
3649 * @param pSelReg Pointer to the segment selector.
3650 * @param pCtx Pointer to the guest-CPU context.
3651 *
3652 * @remarks No-long-jump zone!!!
3653 */
3654static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3655 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3656{
3657 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3658 AssertRCReturn(rc, rc);
3659 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3660 AssertRCReturn(rc, rc);
3661 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3662 AssertRCReturn(rc, rc);
3663
3664 uint32_t u32Access = pSelReg->Attr.u;
3665 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3666 {
3667 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3668 u32Access = 0xf3;
3669 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3670 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3671 }
3672 else
3673 {
3674 /*
3675 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3676 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3677 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3678 * loaded in protected-mode have their attribute as 0.
3679 */
3680 if (!u32Access)
3681 u32Access = X86DESCATTR_UNUSABLE;
3682 }
3683
3684 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3685 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3686 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3687
3688 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3689 AssertRCReturn(rc, rc);
3690 return rc;
3691}
3692
3693
3694/**
3695 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3696 * into the guest-state area in the VMCS.
3697 *
3698 * @returns VBox status code.
3699 * @param pVM Pointer to the VM.
3700 * @param pVCPU Pointer to the VMCPU.
3701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3702 * out-of-sync. Make sure to update the required fields
3703 * before using them.
3704 *
3705 * @remarks ASSUMES CR0 is up to date (strict builds validation).
3706 * @remarks No-long-jump zone!!!
3707 */
3708static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3709{
3710 int rc = VERR_INTERNAL_ERROR_5;
3711 PVM pVM = pVCpu->CTX_SUFF(pVM);
3712
3713 /*
3714 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
3715 */
3716 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3717 {
3718 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3719 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3720 {
3721 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3722 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3723 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3724 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3725 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3726 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3727 }
3728
3729#ifdef VBOX_WITH_REM
3730 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3731 {
3732 Assert(pVM->hm.s.vmx.pRealModeTSS);
3733 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3734 if ( pVCpu->hm.s.vmx.fWasInRealMode
3735 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3736 {
3737 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3738 in real-mode (e.g. OpenBSD 4.0) */
3739 REMFlushTBs(pVM);
3740 Log4(("Load: Switch to protected mode detected!\n"));
3741 pVCpu->hm.s.vmx.fWasInRealMode = false;
3742 }
3743 }
3744#endif
3745 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3746 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3747 AssertRCReturn(rc, rc);
3748 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3749 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3750 AssertRCReturn(rc, rc);
3751 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3752 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3753 AssertRCReturn(rc, rc);
3754 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3755 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3756 AssertRCReturn(rc, rc);
3757 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3758 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3759 AssertRCReturn(rc, rc);
3760 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3761 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3762 AssertRCReturn(rc, rc);
3763
3764 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3765 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3766#ifdef VBOX_STRICT
3767 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3768#endif
3769 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3770 }
3771
3772 /*
3773 * Guest TR.
3774 */
3775 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3776 {
3777 /*
3778 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3779 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3780 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3781 */
3782 uint16_t u16Sel = 0;
3783 uint32_t u32Limit = 0;
3784 uint64_t u64Base = 0;
3785 uint32_t u32AccessRights = 0;
3786
3787 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3788 {
3789 u16Sel = pMixedCtx->tr.Sel;
3790 u32Limit = pMixedCtx->tr.u32Limit;
3791 u64Base = pMixedCtx->tr.u64Base;
3792 u32AccessRights = pMixedCtx->tr.Attr.u;
3793 }
3794 else
3795 {
3796 Assert(pVM->hm.s.vmx.pRealModeTSS);
3797 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3798
3799 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3800 RTGCPHYS GCPhys;
3801 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3802 AssertRCReturn(rc, rc);
3803
3804 X86DESCATTR DescAttr;
3805 DescAttr.u = 0;
3806 DescAttr.n.u1Present = 1;
3807 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3808
3809 u16Sel = 0;
3810 u32Limit = HM_VTX_TSS_SIZE;
3811 u64Base = GCPhys; /* in real-mode phys = virt. */
3812 u32AccessRights = DescAttr.u;
3813 }
3814
3815 /* Validate. */
3816 Assert(!(u16Sel & RT_BIT(2)));
3817 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3818 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3819 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3820 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3821 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3822 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3823 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3824 Assert( (u32Limit & 0xfff) == 0xfff
3825 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3826 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3827 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3828
3829 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3830 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3831 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3832 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3833
3834 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3835 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3836 }
3837
3838 /*
3839 * Guest GDTR.
3840 */
3841 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3842 {
3843 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3844 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3845
3846 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3847 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3848 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3849 }
3850
3851 /*
3852 * Guest LDTR.
3853 */
3854 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3855 {
3856 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3857 uint32_t u32Access = 0;
3858 if (!pMixedCtx->ldtr.Attr.u)
3859 u32Access = X86DESCATTR_UNUSABLE;
3860 else
3861 u32Access = pMixedCtx->ldtr.Attr.u;
3862
3863 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3864 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3865 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3866 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3867
3868 /* Validate. */
3869 if (!(u32Access & X86DESCATTR_UNUSABLE))
3870 {
3871 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3872 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3873 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3874 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3875 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3876 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3877 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3878 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3879 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3880 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3881 }
3882
3883 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3884 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3885 }
3886
3887 /*
3888 * Guest IDTR.
3889 */
3890 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3891 {
3892 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3893 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3894
3895 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3896 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3897 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3898 }
3899
3900 return VINF_SUCCESS;
3901}
3902
3903
3904/**
3905 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3906 * areas. These MSRs will automatically be loaded to the host CPU on every
3907 * successful VM entry and stored from the host CPU on every successful VM exit.
3908 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
3909 *
3910 * @returns VBox status code.
3911 * @param pVCpu Pointer to the VMCPU.
3912 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3913 * out-of-sync. Make sure to update the required fields
3914 * before using them.
3915 *
3916 * @remarks No-long-jump zone!!!
3917 */
3918static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3919{
3920 AssertPtr(pVCpu);
3921 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3922
3923 /*
3924 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3925 */
3926 int rc = VINF_SUCCESS;
3927 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3928 {
3929#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3930 PVM pVM = pVCpu->CTX_SUFF(pVM);
3931 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3932 uint32_t cGuestMsrs = 0;
3933
3934 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3935 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3936 * when the guest really is in 64-bit mode. */
3937 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3938 if (fSupportsLongMode)
3939 {
3940 pGuestMsr->u32Msr = MSR_K8_LSTAR;
3941 pGuestMsr->u32Reserved = 0;
3942 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3943 pGuestMsr++; cGuestMsrs++;
3944 pGuestMsr->u32Msr = MSR_K6_STAR;
3945 pGuestMsr->u32Reserved = 0;
3946 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3947 pGuestMsr++; cGuestMsrs++;
3948 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
3949 pGuestMsr->u32Reserved = 0;
3950 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3951 pGuestMsr++; cGuestMsrs++;
3952 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
3953 pGuestMsr->u32Reserved = 0;
3954 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3955 pGuestMsr++; cGuestMsrs++;
3956 }
3957
3958 /*
3959 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3960 * load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3961 */
3962 if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3963 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3964 {
3965 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
3966 pGuestMsr->u32Reserved = 0;
3967 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3968 AssertRCReturn(rc, rc);
3969 pGuestMsr++; cGuestMsrs++;
3970 }
3971
3972 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3973 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
3974 {
3975 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3976 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
3977 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3978 }
3979
3980 /* Update the VCPU's copy of the guest MSR count. */
3981 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3982 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3983 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3984#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3985
3986 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3987 }
3988
3989 /*
3990 * Guest Sysenter MSRs.
3991 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3992 * VM-exits on WRMSRs for these MSRs.
3993 */
3994 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3995 {
3996 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3997 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3998 }
3999 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4000 {
4001 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4002 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
4003 }
4004 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4005 {
4006 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4007 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
4008 }
4009
4010 return rc;
4011}
4012
4013
4014/**
4015 * Loads the guest activity state into the guest-state area in the VMCS.
4016 *
4017 * @returns VBox status code.
4018 * @param pVCpu Pointer to the VMCPU.
4019 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4020 * out-of-sync. Make sure to update the required fields
4021 * before using them.
4022 *
4023 * @remarks No-long-jump zone!!!
4024 */
4025static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4026{
4027 /** @todo See if we can make use of other states, e.g.
4028 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4029 int rc = VINF_SUCCESS;
4030 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
4031 {
4032 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4033 AssertRCReturn(rc, rc);
4034 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
4035 }
4036 return rc;
4037}
4038
4039
4040/**
4041 * Sets up the appropriate function to run guest code.
4042 *
4043 * @returns VBox status code.
4044 * @param pVCpu Pointer to the VMCPU.
4045 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4046 * out-of-sync. Make sure to update the required fields
4047 * before using them.
4048 *
4049 * @remarks No-long-jump zone!!!
4050 */
4051static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4052{
4053 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4054 {
4055#ifndef VBOX_ENABLE_64_BITS_GUESTS
4056 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4057#endif
4058 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4059#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4060 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4061 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4062 {
4063 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4064 /** @todo this isn't necessary, but I'm still seeing tripple faults. */
4065 VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
4066 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_EXIT_CTLS;
4067 }
4068#else
4069 /* 64-bit host or hybrid host. */
4070 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4071#endif
4072 }
4073 else
4074 {
4075 /* Guest is not in long mode, use the 32-bit handler. */
4076#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4077 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4078 {
4079 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4080 /** @todo r=bird: Don't we need to set up the host resume (after
4081 * vmlaunch/vmresume) state here?? I'm forcing a trip to ring-3 now
4082 * in the hope that it will prevent crashing the host. A better
4083 * fix should be found as the guest may be going back and forth
4084 * between 16/32-bit and long mode frequently at times. */
4085 VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
4086 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_EXIT_CTLS;
4087 }
4088#else
4089 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4090#endif
4091 }
4092 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4093 return VINF_SUCCESS;
4094}
4095
4096
4097/**
4098 * Wrapper for running the guest code in VT-x.
4099 *
4100 * @returns VBox strict status code.
4101 * @param pVM Pointer to the VM.
4102 * @param pVCpu Pointer to the VMCPU.
4103 * @param pCtx Pointer to the guest-CPU context.
4104 *
4105 * @remarks No-long-jump zone!!!
4106 */
4107DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4108{
4109 /*
4110 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4111 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4112 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4113 */
4114 const bool fResumeVM = !!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4115 /** @todo Add stats for resume vs launch. */
4116#ifdef VBOX_WITH_KERNEL_USING_XMM
4117 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4118#else
4119 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4120#endif
4121}
4122
4123
4124/**
4125 * Reports world-switch error and dumps some useful debug info.
4126 *
4127 * @param pVM Pointer to the VM.
4128 * @param pVCpu Pointer to the VMCPU.
4129 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4130 * @param pCtx Pointer to the guest-CPU context.
4131 * @param pVmxTransient Pointer to the VMX transient structure (only
4132 * exitReason updated).
4133 */
4134static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4135{
4136 Assert(pVM);
4137 Assert(pVCpu);
4138 Assert(pCtx);
4139 Assert(pVmxTransient);
4140 HMVMX_ASSERT_PREEMPT_SAFE();
4141
4142 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4143 switch (rcVMRun)
4144 {
4145 case VERR_VMX_INVALID_VMXON_PTR:
4146 AssertFailed();
4147 break;
4148 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4149 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4150 {
4151 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4152 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4153 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4154 AssertRC(rc);
4155
4156 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4157 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4158 Cannot do it here as we may have been long preempted. */
4159
4160#ifdef VBOX_STRICT
4161 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4162 pVmxTransient->uExitReason));
4163 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4164 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4165 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4166 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4167 else
4168 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4169 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4170 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4171
4172 /* VMX control bits. */
4173 uint32_t u32Val;
4174 uint64_t u64Val;
4175 HMVMXHCUINTREG uHCReg;
4176 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4177 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4178 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4179 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4180 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4181 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4182 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4183 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4184 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4185 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4186 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4187 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4188 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4189 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4190 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4191 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4192 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4193 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4194 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4195 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4196 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4197 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4198 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4199 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4200 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4201 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4202 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4203 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4204 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4205 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4206 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4207 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4208 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4209 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4210 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4211 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4212 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4213 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4214 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4215 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4216 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4217 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4218
4219 /* Guest bits. */
4220 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4221 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4222 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4223 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4224 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4225 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4226 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4227 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4228
4229 /* Host bits. */
4230 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4231 Log4(("Host CR0 %#RHr\n", uHCReg));
4232 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4233 Log4(("Host CR3 %#RHr\n", uHCReg));
4234 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4235 Log4(("Host CR4 %#RHr\n", uHCReg));
4236
4237 RTGDTR HostGdtr;
4238 PCX86DESCHC pDesc;
4239 ASMGetGDTR(&HostGdtr);
4240 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4241 Log4(("Host CS %#08x\n", u32Val));
4242 if (u32Val < HostGdtr.cbGdt)
4243 {
4244 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4245 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4246 }
4247
4248 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4249 Log4(("Host DS %#08x\n", u32Val));
4250 if (u32Val < HostGdtr.cbGdt)
4251 {
4252 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4253 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4254 }
4255
4256 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4257 Log4(("Host ES %#08x\n", u32Val));
4258 if (u32Val < HostGdtr.cbGdt)
4259 {
4260 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4261 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4262 }
4263
4264 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4265 Log4(("Host FS %#08x\n", u32Val));
4266 if (u32Val < HostGdtr.cbGdt)
4267 {
4268 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4269 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4270 }
4271
4272 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4273 Log4(("Host GS %#08x\n", u32Val));
4274 if (u32Val < HostGdtr.cbGdt)
4275 {
4276 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4277 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4278 }
4279
4280 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4281 Log4(("Host SS %#08x\n", u32Val));
4282 if (u32Val < HostGdtr.cbGdt)
4283 {
4284 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4285 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4286 }
4287
4288 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4289 Log4(("Host TR %#08x\n", u32Val));
4290 if (u32Val < HostGdtr.cbGdt)
4291 {
4292 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4293 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4294 }
4295
4296 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4297 Log4(("Host TR Base %#RHv\n", uHCReg));
4298 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4299 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4300 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4301 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4302 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4303 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4304 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4305 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4306 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4307 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4308 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4309 Log4(("Host RSP %#RHv\n", uHCReg));
4310 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4311 Log4(("Host RIP %#RHv\n", uHCReg));
4312# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4313 if (HMVMX_IS_64BIT_HOST_MODE())
4314 {
4315 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4316 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4317 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4318 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4319 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4320 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4321 }
4322# endif
4323#endif /* VBOX_STRICT */
4324 break;
4325 }
4326
4327 default:
4328 /* Impossible */
4329 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4330 break;
4331 }
4332 NOREF(pVM);
4333}
4334
4335
4336#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4337#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4338# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4339#endif
4340#ifdef VBOX_STRICT
4341static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4342{
4343 switch (idxField)
4344 {
4345 case VMX_VMCS_GUEST_RIP:
4346 case VMX_VMCS_GUEST_RSP:
4347 case VMX_VMCS_GUEST_SYSENTER_EIP:
4348 case VMX_VMCS_GUEST_SYSENTER_ESP:
4349 case VMX_VMCS_GUEST_GDTR_BASE:
4350 case VMX_VMCS_GUEST_IDTR_BASE:
4351 case VMX_VMCS_GUEST_CS_BASE:
4352 case VMX_VMCS_GUEST_DS_BASE:
4353 case VMX_VMCS_GUEST_ES_BASE:
4354 case VMX_VMCS_GUEST_FS_BASE:
4355 case VMX_VMCS_GUEST_GS_BASE:
4356 case VMX_VMCS_GUEST_SS_BASE:
4357 case VMX_VMCS_GUEST_LDTR_BASE:
4358 case VMX_VMCS_GUEST_TR_BASE:
4359 case VMX_VMCS_GUEST_CR3:
4360 return true;
4361 }
4362 return false;
4363}
4364
4365static bool hmR0VmxIsValidReadField(uint32_t idxField)
4366{
4367 switch (idxField)
4368 {
4369 /* Read-only fields. */
4370 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4371 return true;
4372 }
4373 /* Remaining readable fields should also be writable. */
4374 return hmR0VmxIsValidWriteField(idxField);
4375}
4376#endif /* VBOX_STRICT */
4377
4378
4379/**
4380 * Executes the specified handler in 64-bit mode.
4381 *
4382 * @returns VBox status code.
4383 * @param pVM Pointer to the VM.
4384 * @param pVCpu Pointer to the VMCPU.
4385 * @param pCtx Pointer to the guest CPU context.
4386 * @param enmOp The operation to perform.
4387 * @param cbParam Number of parameters.
4388 * @param paParam Array of 32-bit parameters.
4389 */
4390VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4391 uint32_t *paParam)
4392{
4393 int rc, rc2;
4394 PHMGLOBALCPUINFO pCpu;
4395 RTHCPHYS HCPhysCpuPage;
4396 RTCCUINTREG uOldEflags;
4397
4398 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4399 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4400 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4401 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4402
4403#ifdef VBOX_STRICT
4404 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4405 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4406
4407 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4408 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4409#endif
4410
4411 /* Disable interrupts. */
4412 uOldEflags = ASMIntDisableFlags();
4413
4414#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4415 RTCPUID idHostCpu = RTMpCpuId();
4416 CPUMR0SetLApic(pVCpu, idHostCpu);
4417#endif
4418
4419 pCpu = HMR0GetCurrentCpu();
4420 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4421
4422 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4423 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4424
4425 /* Leave VMX Root Mode. */
4426 VMXDisable();
4427
4428 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4429
4430 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4431 CPUMSetHyperEIP(pVCpu, enmOp);
4432 for (int i = (int)cbParam - 1; i >= 0; i--)
4433 CPUMPushHyper(pVCpu, paParam[i]);
4434
4435 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4436
4437 /* Call the switcher. */
4438 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4439 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4440
4441 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4442 /* Make sure the VMX instructions don't cause #UD faults. */
4443 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4444
4445 /* Re-enter VMX Root Mode */
4446 rc2 = VMXEnable(HCPhysCpuPage);
4447 if (RT_FAILURE(rc2))
4448 {
4449 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4450 ASMSetFlags(uOldEflags);
4451 return rc2;
4452 }
4453
4454 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4455 AssertRC(rc2);
4456 Assert(!(ASMGetFlags() & X86_EFL_IF));
4457 ASMSetFlags(uOldEflags);
4458 return rc;
4459}
4460
4461
4462/**
4463 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4464 * supporting 64-bit guests.
4465 *
4466 * @returns VBox status code.
4467 * @param fResume Whether to VMLAUNCH or VMRESUME.
4468 * @param pCtx Pointer to the guest-CPU context.
4469 * @param pCache Pointer to the VMCS cache.
4470 * @param pVM Pointer to the VM.
4471 * @param pVCpu Pointer to the VMCPU.
4472 */
4473DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4474{
4475 uint32_t aParam[6];
4476 PHMGLOBALCPUINFO pCpu = NULL;
4477 RTHCPHYS HCPhysCpuPage = 0;
4478 int rc = VERR_INTERNAL_ERROR_5;
4479
4480 pCpu = HMR0GetCurrentCpu();
4481 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4482
4483#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4484 pCache->uPos = 1;
4485 pCache->interPD = PGMGetInterPaeCR3(pVM);
4486 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4487#endif
4488
4489#ifdef VBOX_STRICT
4490 pCache->TestIn.HCPhysCpuPage = 0;
4491 pCache->TestIn.HCPhysVmcs = 0;
4492 pCache->TestIn.pCache = 0;
4493 pCache->TestOut.HCPhysVmcs = 0;
4494 pCache->TestOut.pCache = 0;
4495 pCache->TestOut.pCtx = 0;
4496 pCache->TestOut.eflags = 0;
4497#endif
4498
4499 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4500 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4501 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4502 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4503 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4504 aParam[5] = 0;
4505
4506#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4507 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4508 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4509#endif
4510 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4511
4512#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4513 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4514 Assert(pCtx->dr[4] == 10);
4515 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4516#endif
4517
4518#ifdef VBOX_STRICT
4519 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4520 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4521 pVCpu->hm.s.vmx.HCPhysVmcs));
4522 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4523 pCache->TestOut.HCPhysVmcs));
4524 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4525 pCache->TestOut.pCache));
4526 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4527 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4528 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4529 pCache->TestOut.pCtx));
4530 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4531#endif
4532 return rc;
4533}
4534
4535
4536/**
4537 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4538 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4539 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4540 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4541 *
4542 * @returns VBox status code.
4543 * @param pVM Pointer to the VM.
4544 * @param pVCpu Pointer to the VMCPU.
4545 */
4546static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4547{
4548#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4549{ \
4550 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4551 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4552 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4553 ++cReadFields; \
4554}
4555
4556 AssertPtr(pVM);
4557 AssertPtr(pVCpu);
4558 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4559 uint32_t cReadFields = 0;
4560
4561 /*
4562 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4563 * and serve to indicate exceptions to the rules.
4564 */
4565
4566 /* Guest-natural selector base fields. */
4567#if 0
4568 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4569 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4570 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4571#endif
4572 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4573 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4574 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4575 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4576 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4577 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4578 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4579 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4580 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4581 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4582 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4583 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4584#if 0
4585 /* Unused natural width guest-state fields. */
4586 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4587 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4588#endif
4589 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4590 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4591
4592 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4593#if 0
4594 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4595 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4596 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4597 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4598 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4599 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4600 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4601 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4602 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4603#endif
4604
4605 /* Natural width guest-state fields. */
4606 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4607#if 0
4608 /* Currently unused field. */
4609 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4610#endif
4611
4612 if (pVM->hm.s.fNestedPaging)
4613 {
4614 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4615 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4616 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4617 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4618 }
4619 else
4620 {
4621 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4622 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4623 }
4624
4625#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4626 return VINF_SUCCESS;
4627}
4628
4629
4630/**
4631 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4632 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4633 * darwin, running 64-bit guests).
4634 *
4635 * @returns VBox status code.
4636 * @param pVCpu Pointer to the VMCPU.
4637 * @param idxField The VMCS field encoding.
4638 * @param u64Val 16, 32 or 64 bits value.
4639 */
4640VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4641{
4642 int rc;
4643 switch (idxField)
4644 {
4645 /*
4646 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4647 */
4648 /* 64-bit Control fields. */
4649 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4650 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4651 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4652 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4653 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4654 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4655 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4656 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4657 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4658 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4659 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4660 case VMX_VMCS64_CTRL_EPTP_FULL:
4661 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4662 /* 64-bit Guest-state fields. */
4663 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4664 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4665 case VMX_VMCS64_GUEST_PAT_FULL:
4666 case VMX_VMCS64_GUEST_EFER_FULL:
4667 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4668 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4669 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4670 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4671 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4672 /* 64-bit Host-state fields. */
4673 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4674 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4675 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4676 {
4677 rc = VMXWriteVmcs32(idxField, u64Val);
4678 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4679 break;
4680 }
4681
4682 /*
4683 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4684 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4685 */
4686 /* Natural-width Guest-state fields. */
4687 case VMX_VMCS_GUEST_CR3:
4688 case VMX_VMCS_GUEST_ES_BASE:
4689 case VMX_VMCS_GUEST_CS_BASE:
4690 case VMX_VMCS_GUEST_SS_BASE:
4691 case VMX_VMCS_GUEST_DS_BASE:
4692 case VMX_VMCS_GUEST_FS_BASE:
4693 case VMX_VMCS_GUEST_GS_BASE:
4694 case VMX_VMCS_GUEST_LDTR_BASE:
4695 case VMX_VMCS_GUEST_TR_BASE:
4696 case VMX_VMCS_GUEST_GDTR_BASE:
4697 case VMX_VMCS_GUEST_IDTR_BASE:
4698 case VMX_VMCS_GUEST_RSP:
4699 case VMX_VMCS_GUEST_RIP:
4700 case VMX_VMCS_GUEST_SYSENTER_ESP:
4701 case VMX_VMCS_GUEST_SYSENTER_EIP:
4702 {
4703 if (!(u64Val >> 32))
4704 {
4705 /* If this field is 64-bit, VT-x will zero out the top bits. */
4706 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4707 }
4708 else
4709 {
4710 /* Assert that only the 32->64 switcher case should ever come here. */
4711 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4712 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4713 }
4714 break;
4715 }
4716
4717 default:
4718 {
4719 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4720 rc = VERR_INVALID_PARAMETER;
4721 break;
4722 }
4723 }
4724 AssertRCReturn(rc, rc);
4725 return rc;
4726}
4727
4728
4729/**
4730 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4731 * hosts (except darwin) for 64-bit guests.
4732 *
4733 * @param pVCpu Pointer to the VMCPU.
4734 * @param idxField The VMCS field encoding.
4735 * @param u64Val 16, 32 or 64 bits value.
4736 */
4737VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4738{
4739 AssertPtr(pVCpu);
4740 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4741
4742 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4743 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4744
4745 /* Make sure there are no duplicates. */
4746 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4747 {
4748 if (pCache->Write.aField[i] == idxField)
4749 {
4750 pCache->Write.aFieldVal[i] = u64Val;
4751 return VINF_SUCCESS;
4752 }
4753 }
4754
4755 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4756 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4757 pCache->Write.cValidEntries++;
4758 return VINF_SUCCESS;
4759}
4760
4761/* Enable later when the assembly code uses these as callbacks. */
4762#if 0
4763/*
4764 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4765 *
4766 * @param pVCpu Pointer to the VMCPU.
4767 * @param pCache Pointer to the VMCS cache.
4768 *
4769 * @remarks No-long-jump zone!!!
4770 */
4771VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4772{
4773 AssertPtr(pCache);
4774 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4775 {
4776 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4777 AssertRC(rc);
4778 }
4779 pCache->Write.cValidEntries = 0;
4780}
4781
4782
4783/**
4784 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
4785 *
4786 * @param pVCpu Pointer to the VMCPU.
4787 * @param pCache Pointer to the VMCS cache.
4788 *
4789 * @remarks No-long-jump zone!!!
4790 */
4791VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4792{
4793 AssertPtr(pCache);
4794 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4795 {
4796 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4797 AssertRC(rc);
4798 }
4799}
4800#endif
4801#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4802
4803
4804/**
4805 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4806 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4807 * timer.
4808 *
4809 * @returns VBox status code.
4810 * @param pVCpu Pointer to the VMCPU.
4811 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4812 * out-of-sync. Make sure to update the required fields
4813 * before using them.
4814 * @remarks No-long-jump zone!!!
4815 */
4816static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4817{
4818 int rc = VERR_INTERNAL_ERROR_5;
4819 bool fOffsettedTsc = false;
4820 PVM pVM = pVCpu->CTX_SUFF(pVM);
4821 if (pVM->hm.s.vmx.fUsePreemptTimer)
4822 {
4823 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4824
4825 /* Make sure the returned values have sane upper and lower boundaries. */
4826 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4827 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4828 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4829 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4830
4831 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4832 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4833 }
4834 else
4835 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4836
4837 if (fOffsettedTsc)
4838 {
4839 uint64_t u64CurTSC = ASMReadTSC();
4840 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4841 {
4842 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4843 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4844
4845 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4846 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4847 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4848 }
4849 else
4850 {
4851 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4852 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4853 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4854 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4855 }
4856 }
4857 else
4858 {
4859 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4860 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4861 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4862 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4863 }
4864}
4865
4866
4867/**
4868 * Determines if an exception is a contributory exception. Contributory
4869 * exceptions are ones which can cause double-faults. Page-fault is
4870 * intentionally not included here as it's a conditional contributory exception.
4871 *
4872 * @returns true if the exception is contributory, false otherwise.
4873 * @param uVector The exception vector.
4874 */
4875DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4876{
4877 switch (uVector)
4878 {
4879 case X86_XCPT_GP:
4880 case X86_XCPT_SS:
4881 case X86_XCPT_NP:
4882 case X86_XCPT_TS:
4883 case X86_XCPT_DE:
4884 return true;
4885 default:
4886 break;
4887 }
4888 return false;
4889}
4890
4891
4892/**
4893 * Sets an event as a pending event to be injected into the guest.
4894 *
4895 * @param pVCpu Pointer to the VMCPU.
4896 * @param u32IntrInfo The VM-entry interruption-information field.
4897 * @param cbInstr The VM-entry instruction length in bytes (for software
4898 * interrupts, exceptions and privileged software
4899 * exceptions).
4900 * @param u32ErrCode The VM-entry exception error code.
4901 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4902 * page-fault.
4903 *
4904 * @remarks Statistics counter assumes this is a guest event being injected or
4905 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4906 * always incremented.
4907 */
4908DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4909 RTGCUINTPTR GCPtrFaultAddress)
4910{
4911 Assert(!pVCpu->hm.s.Event.fPending);
4912 pVCpu->hm.s.Event.fPending = true;
4913 pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4914 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4915 pVCpu->hm.s.Event.cbInstr = cbInstr;
4916 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4917
4918 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4919}
4920
4921
4922/**
4923 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4924 *
4925 * @param pVCpu Pointer to the VMCPU.
4926 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4927 * out-of-sync. Make sure to update the required fields
4928 * before using them.
4929 */
4930DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4931{
4932 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
4933 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4934 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4935 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
4936}
4937
4938
4939/**
4940 * Handle a condition that occurred while delivering an event through the guest
4941 * IDT.
4942 *
4943 * @returns VBox status code (informational error codes included).
4944 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
4945 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4946 * continue execution of the guest which will delivery the #DF.
4947 * @retval VINF_EM_RESET if we detected a triple-fault condition.
4948 *
4949 * @param pVCpu Pointer to the VMCPU.
4950 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4951 * out-of-sync. Make sure to update the required fields
4952 * before using them.
4953 * @param pVmxTransient Pointer to the VMX transient structure.
4954 *
4955 * @remarks No-long-jump zone!!!
4956 */
4957static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4958{
4959 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4960 AssertRCReturn(rc, rc);
4961 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4962 {
4963 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4964 AssertRCReturn(rc, rc);
4965
4966 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4967 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4968 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4969
4970 typedef enum
4971 {
4972 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4973 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4974 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4975 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4976 } VMXREFLECTXCPT;
4977
4978 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4979 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4980 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4981 {
4982 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4983 {
4984 enmReflect = VMXREFLECTXCPT_XCPT;
4985#ifdef VBOX_STRICT
4986 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4987 && uExitVector == X86_XCPT_PF)
4988 {
4989 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4990 }
4991#endif
4992 if ( uExitVector == X86_XCPT_PF
4993 && uIdtVector == X86_XCPT_PF)
4994 {
4995 pVmxTransient->fVectoringPF = true;
4996 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4997 }
4998 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4999 && hmR0VmxIsContributoryXcpt(uExitVector)
5000 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5001 || uIdtVector == X86_XCPT_PF))
5002 {
5003 enmReflect = VMXREFLECTXCPT_DF;
5004 }
5005 else if (uIdtVector == X86_XCPT_DF)
5006 enmReflect = VMXREFLECTXCPT_TF;
5007 }
5008 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5009 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5010 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5011 {
5012 /*
5013 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5014 * (whatever they are) as they reoccur when restarting the instruction.
5015 */
5016 enmReflect = VMXREFLECTXCPT_XCPT;
5017 }
5018 }
5019 else
5020 {
5021 /*
5022 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5023 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5024 * original exception to the guest after handling the VM-exit.
5025 */
5026 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5027 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5028 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5029 {
5030 enmReflect = VMXREFLECTXCPT_XCPT;
5031 }
5032 }
5033
5034 switch (enmReflect)
5035 {
5036 case VMXREFLECTXCPT_XCPT:
5037 {
5038 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5039 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5040 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5041
5042 uint32_t u32ErrCode = 0;
5043 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5044 {
5045 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5046 AssertRCReturn(rc, rc);
5047 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5048 }
5049
5050 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5051 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5052 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5053 rc = VINF_SUCCESS;
5054 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5055 pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
5056
5057 break;
5058 }
5059
5060 case VMXREFLECTXCPT_DF:
5061 {
5062 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5063 rc = VINF_HM_DOUBLE_FAULT;
5064 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5065 pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
5066
5067 break;
5068 }
5069
5070 case VMXREFLECTXCPT_TF:
5071 {
5072 rc = VINF_EM_RESET;
5073 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5074 uExitVector));
5075 break;
5076 }
5077
5078 default:
5079 Assert(rc == VINF_SUCCESS);
5080 break;
5081 }
5082 }
5083 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5084 return rc;
5085}
5086
5087
5088/**
5089 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5090 *
5091 * @returns VBox status code.
5092 * @param pVCpu Pointer to the VMCPU.
5093 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5094 * out-of-sync. Make sure to update the required fields
5095 * before using them.
5096 *
5097 * @remarks No-long-jump zone!!!
5098 */
5099static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5100{
5101 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5102 {
5103 uint32_t uVal = 0;
5104 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5105 AssertRCReturn(rc, rc);
5106 uint32_t uShadow = 0;
5107 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5108 AssertRCReturn(rc, rc);
5109
5110 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5111 CPUMSetGuestCR0(pVCpu, uVal);
5112 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5113 }
5114 return VINF_SUCCESS;
5115}
5116
5117
5118/**
5119 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5120 *
5121 * @returns VBox status code.
5122 * @param pVCpu Pointer to the VMCPU.
5123 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5124 * out-of-sync. Make sure to update the required fields
5125 * before using them.
5126 *
5127 * @remarks No-long-jump zone!!!
5128 */
5129static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5130{
5131 int rc = VINF_SUCCESS;
5132 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5133 {
5134 uint32_t uVal = 0;
5135 uint32_t uShadow = 0;
5136 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5137 AssertRCReturn(rc, rc);
5138 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5139 AssertRCReturn(rc, rc);
5140
5141 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5142 CPUMSetGuestCR4(pVCpu, uVal);
5143 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5144 }
5145 return rc;
5146}
5147
5148
5149/**
5150 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5151 *
5152 * @returns VBox status code.
5153 * @param pVCpu Pointer to the VMCPU.
5154 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5155 * out-of-sync. Make sure to update the required fields
5156 * before using them.
5157 *
5158 * @remarks No-long-jump zone!!!
5159 */
5160static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5161{
5162 int rc = VINF_SUCCESS;
5163 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5164 {
5165 uint64_t u64Val = 0;
5166 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5167 AssertRCReturn(rc, rc);
5168
5169 pMixedCtx->rip = u64Val;
5170 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5171 }
5172 return rc;
5173}
5174
5175
5176/**
5177 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5178 *
5179 * @returns VBox status code.
5180 * @param pVCpu Pointer to the VMCPU.
5181 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5182 * out-of-sync. Make sure to update the required fields
5183 * before using them.
5184 *
5185 * @remarks No-long-jump zone!!!
5186 */
5187static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5188{
5189 int rc = VINF_SUCCESS;
5190 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5191 {
5192 uint64_t u64Val = 0;
5193 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5194 AssertRCReturn(rc, rc);
5195
5196 pMixedCtx->rsp = u64Val;
5197 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5198 }
5199 return rc;
5200}
5201
5202
5203/**
5204 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5205 *
5206 * @returns VBox status code.
5207 * @param pVCpu Pointer to the VMCPU.
5208 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5209 * out-of-sync. Make sure to update the required fields
5210 * before using them.
5211 *
5212 * @remarks No-long-jump zone!!!
5213 */
5214static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5215{
5216 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5217 {
5218 uint32_t uVal = 0;
5219 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5220 AssertRCReturn(rc, rc);
5221
5222 pMixedCtx->eflags.u32 = uVal;
5223 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5224 {
5225 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5226 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5227
5228 pMixedCtx->eflags.Bits.u1VM = 0;
5229 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5230 }
5231
5232 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5233 }
5234 return VINF_SUCCESS;
5235}
5236
5237
5238/**
5239 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5240 * guest-CPU context.
5241 */
5242DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5243{
5244 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5245 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5246 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5247 return rc;
5248}
5249
5250
5251/**
5252 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5253 * from the guest-state area in the VMCS.
5254 *
5255 * @param pVCpu Pointer to the VMCPU.
5256 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5257 * out-of-sync. Make sure to update the required fields
5258 * before using them.
5259 *
5260 * @remarks No-long-jump zone!!!
5261 */
5262static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5263{
5264 uint32_t uIntrState = 0;
5265 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5266 AssertRC(rc);
5267
5268 if (!uIntrState)
5269 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5270 else
5271 {
5272 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5273 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5274 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5275 AssertRC(rc);
5276 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5277 AssertRC(rc);
5278
5279 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5280 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5281 }
5282}
5283
5284
5285/**
5286 * Saves the guest's activity state.
5287 *
5288 * @returns VBox status code.
5289 * @param pVCpu Pointer to the VMCPU.
5290 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5291 * out-of-sync. Make sure to update the required fields
5292 * before using them.
5293 *
5294 * @remarks No-long-jump zone!!!
5295 */
5296static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5297{
5298 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5299 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5300 return VINF_SUCCESS;
5301}
5302
5303
5304/**
5305 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5306 * the current VMCS into the guest-CPU context.
5307 *
5308 * @returns VBox status code.
5309 * @param pVCpu Pointer to the VMCPU.
5310 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5311 * out-of-sync. Make sure to update the required fields
5312 * before using them.
5313 *
5314 * @remarks No-long-jump zone!!!
5315 */
5316static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5317{
5318 int rc = VINF_SUCCESS;
5319 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5320 {
5321 uint32_t u32Val = 0;
5322 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5323 pMixedCtx->SysEnter.cs = u32Val;
5324 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5325 }
5326
5327 uint64_t u64Val = 0;
5328 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5329 {
5330 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5331 pMixedCtx->SysEnter.eip = u64Val;
5332 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5333 }
5334 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5335 {
5336 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5337 pMixedCtx->SysEnter.esp = u64Val;
5338 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5339 }
5340 return rc;
5341}
5342
5343
5344/**
5345 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5346 * context.
5347 *
5348 * @returns VBox status code.
5349 * @param pVCpu Pointer to the VMCPU.
5350 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5351 * out-of-sync. Make sure to update the required fields
5352 * before using them.
5353 *
5354 * @remarks No-long-jump zone!!!
5355 */
5356static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5357{
5358 int rc = VINF_SUCCESS;
5359 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5360 {
5361 uint64_t u64Val = 0;
5362 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5363 pMixedCtx->fs.u64Base = u64Val;
5364 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5365 }
5366 return rc;
5367}
5368
5369
5370/**
5371 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5372 * context.
5373 *
5374 * @returns VBox status code.
5375 * @param pVCpu Pointer to the VMCPU.
5376 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5377 * out-of-sync. Make sure to update the required fields
5378 * before using them.
5379 *
5380 * @remarks No-long-jump zone!!!
5381 */
5382static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5383{
5384 int rc = VINF_SUCCESS;
5385 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5386 {
5387 uint64_t u64Val = 0;
5388 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5389 pMixedCtx->gs.u64Base = u64Val;
5390 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5391 }
5392 return rc;
5393}
5394
5395
5396/**
5397 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5398 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5399 * and TSC_AUX.
5400 *
5401 * @returns VBox status code.
5402 * @param pVCpu Pointer to the VMCPU.
5403 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5404 * out-of-sync. Make sure to update the required fields
5405 * before using them.
5406 *
5407 * @remarks No-long-jump zone!!!
5408 */
5409static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5410{
5411 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5412 return VINF_SUCCESS;
5413
5414#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5415 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5416 {
5417 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5418 pMsr += i;
5419 switch (pMsr->u32Msr)
5420 {
5421 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5422 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5423 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5424 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5425 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5426 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5427 default:
5428 {
5429 AssertFailed();
5430 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5431 }
5432 }
5433 }
5434#endif
5435
5436 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5437 return VINF_SUCCESS;
5438}
5439
5440
5441/**
5442 * Saves the guest control registers from the current VMCS into the guest-CPU
5443 * context.
5444 *
5445 * @returns VBox status code.
5446 * @param pVCpu Pointer to the VMCPU.
5447 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5448 * out-of-sync. Make sure to update the required fields
5449 * before using them.
5450 *
5451 * @remarks No-long-jump zone!!!
5452 */
5453static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5454{
5455 /* Guest CR0. Guest FPU. */
5456 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5457 AssertRCReturn(rc, rc);
5458
5459 /* Guest CR4. */
5460 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5461 AssertRCReturn(rc, rc);
5462
5463 /* Guest CR2 - updated always during the world-switch or in #PF. */
5464 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5465 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5466 {
5467 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5468 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5469
5470 PVM pVM = pVCpu->CTX_SUFF(pVM);
5471 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5472 || ( pVM->hm.s.fNestedPaging
5473 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5474 {
5475 uint64_t u64Val = 0;
5476 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5477 if (pMixedCtx->cr3 != u64Val)
5478 {
5479 CPUMSetGuestCR3(pVCpu, u64Val);
5480 if (VMMRZCallRing3IsEnabled(pVCpu))
5481 {
5482 PGMUpdateCR3(pVCpu, u64Val);
5483 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5484 }
5485 else
5486 {
5487 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5488 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5489 }
5490 }
5491
5492 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5493 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5494 {
5495 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5496 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5497 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5498 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5499
5500 if (VMMRZCallRing3IsEnabled(pVCpu))
5501 {
5502 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5503 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5504 }
5505 else
5506 {
5507 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5508 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5509 }
5510 }
5511 }
5512
5513 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5514 }
5515
5516 /*
5517 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5518 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5519 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5520 *
5521 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5522 */
5523 if (VMMRZCallRing3IsEnabled(pVCpu))
5524 {
5525 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5526 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5527
5528 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5529 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5530
5531 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5532 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5533 }
5534
5535 return rc;
5536}
5537
5538
5539/**
5540 * Reads a guest segment register from the current VMCS into the guest-CPU
5541 * context.
5542 *
5543 * @returns VBox status code.
5544 * @param pVCpu Pointer to the VMCPU.
5545 * @param idxSel Index of the selector in the VMCS.
5546 * @param idxLimit Index of the segment limit in the VMCS.
5547 * @param idxBase Index of the segment base in the VMCS.
5548 * @param idxAccess Index of the access rights of the segment in the VMCS.
5549 * @param pSelReg Pointer to the segment selector.
5550 *
5551 * @remarks No-long-jump zone!!!
5552 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5553 * macro as that takes care of whether to read from the VMCS cache or
5554 * not.
5555 */
5556DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5557 PCPUMSELREG pSelReg)
5558{
5559 uint32_t u32Val = 0;
5560 int rc = VMXReadVmcs32(idxSel, &u32Val);
5561 AssertRCReturn(rc, rc);
5562 pSelReg->Sel = (uint16_t)u32Val;
5563 pSelReg->ValidSel = (uint16_t)u32Val;
5564 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5565
5566 rc = VMXReadVmcs32(idxLimit, &u32Val);
5567 AssertRCReturn(rc, rc);
5568 pSelReg->u32Limit = u32Val;
5569
5570 uint64_t u64Val = 0;
5571 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5572 AssertRCReturn(rc, rc);
5573 pSelReg->u64Base = u64Val;
5574
5575 rc = VMXReadVmcs32(idxAccess, &u32Val);
5576 AssertRCReturn(rc, rc);
5577 pSelReg->Attr.u = u32Val;
5578
5579 /*
5580 * If VT-x marks the segment as unusable, most other bits remain undefined:
5581 * - For CS the L, D and G bits have meaning.
5582 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5583 * - For the remaining data segments no bits are defined.
5584 *
5585 * The present bit and the unusable bit has been observed to be set at the
5586 * same time (the selector was supposed to invalid as we started executing
5587 * a V8086 interrupt in ring-0).
5588 *
5589 * What should be important for the rest of the VBox code that the P bit is
5590 * cleared. Some of the other VBox code recognizes the unusable bit, but
5591 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5592 * safe side here, we'll strip off P and other bits we don't care about. If
5593 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5594 *
5595 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5596 */
5597 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5598 {
5599 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5600
5601 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5602 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5603 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5604
5605 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5606#ifdef DEBUG_bird
5607 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5608 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5609 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5610#endif
5611 }
5612 return VINF_SUCCESS;
5613}
5614
5615
5616#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5617# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5618 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5619 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5620#else
5621# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5622 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5623 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5624#endif
5625
5626
5627/**
5628 * Saves the guest segment registers from the current VMCS into the guest-CPU
5629 * context.
5630 *
5631 * @returns VBox status code.
5632 * @param pVCpu Pointer to the VMCPU.
5633 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5634 * out-of-sync. Make sure to update the required fields
5635 * before using them.
5636 *
5637 * @remarks No-long-jump zone!!!
5638 */
5639static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5640{
5641 /* Guest segment registers. */
5642 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5643 {
5644 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5645 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5646 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5647 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5648 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5649 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5650 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5651
5652 /* Restore segment attributes for real-on-v86 mode hack. */
5653 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5654 {
5655 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5656 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5657 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5658 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5659 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5660 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5661 }
5662 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5663 }
5664
5665 return VINF_SUCCESS;
5666}
5667
5668
5669/**
5670 * Saves the guest descriptor table registers and task register from the current
5671 * VMCS into the guest-CPU context.
5672 *
5673 * @returns VBox status code.
5674 * @param pVCpu Pointer to the VMCPU.
5675 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5676 * out-of-sync. Make sure to update the required fields
5677 * before using them.
5678 *
5679 * @remarks No-long-jump zone!!!
5680 */
5681static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5682{
5683 int rc = VINF_SUCCESS;
5684
5685 /* Guest LDTR. */
5686 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5687 {
5688 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5689 AssertRCReturn(rc, rc);
5690 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5691 }
5692
5693 /* Guest GDTR. */
5694 uint64_t u64Val = 0;
5695 uint32_t u32Val = 0;
5696 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5697 {
5698 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5699 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5700 pMixedCtx->gdtr.pGdt = u64Val;
5701 pMixedCtx->gdtr.cbGdt = u32Val;
5702 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
5703 }
5704
5705 /* Guest IDTR. */
5706 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5707 {
5708 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5709 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5710 pMixedCtx->idtr.pIdt = u64Val;
5711 pMixedCtx->idtr.cbIdt = u32Val;
5712 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
5713 }
5714
5715 /* Guest TR. */
5716 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5717 {
5718 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5719 AssertRCReturn(rc, rc);
5720
5721 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5722 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5723 {
5724 rc = VMXLOCAL_READ_SEG(TR, tr);
5725 AssertRCReturn(rc, rc);
5726 }
5727 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
5728 }
5729 return rc;
5730}
5731
5732#undef VMXLOCAL_READ_SEG
5733
5734
5735/**
5736 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5737 * context.
5738 *
5739 * @returns VBox status code.
5740 * @param pVCpu Pointer to the VMCPU.
5741 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5742 * out-of-sync. Make sure to update the required fields
5743 * before using them.
5744 *
5745 * @remarks No-long-jump zone!!!
5746 */
5747static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5748{
5749 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5750 {
5751 if (!CPUMIsHyperDebugStateActive(pVCpu))
5752 {
5753 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5754 uint32_t u32Val;
5755 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5756 pMixedCtx->dr[7] = u32Val;
5757 }
5758
5759 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
5760 }
5761 return VINF_SUCCESS;
5762}
5763
5764
5765/**
5766 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
5767 *
5768 * @returns VBox status code.
5769 * @param pVCpu Pointer to the VMCPU.
5770 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5771 * out-of-sync. Make sure to update the required fields
5772 * before using them.
5773 *
5774 * @remarks No-long-jump zone!!!
5775 */
5776static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5777{
5778 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5779 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
5780 return VINF_SUCCESS;
5781}
5782
5783
5784/**
5785 * Saves the entire guest state from the currently active VMCS into the
5786 * guest-CPU context. This essentially VMREADs all guest-data.
5787 *
5788 * @returns VBox status code.
5789 * @param pVCpu Pointer to the VMCPU.
5790 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5791 * out-of-sync. Make sure to update the required fields
5792 * before using them.
5793 */
5794static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5795{
5796 Assert(pVCpu);
5797 Assert(pMixedCtx);
5798
5799 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5800 return VINF_SUCCESS;
5801
5802 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
5803 again on the ring-3 callback path, there is no real need to. */
5804 if (VMMRZCallRing3IsEnabled(pVCpu))
5805 VMMR0LogFlushDisable(pVCpu);
5806 else
5807 Assert(VMMR0IsLogFlushDisabled(pVCpu));
5808 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5809
5810 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5811 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5812
5813 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5814 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5815
5816 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5817 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5818
5819 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5820 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5821
5822 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5823 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5824
5825 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5826 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5827
5828 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5829 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5830
5831 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5832 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5833
5834 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5835 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5836
5837 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5838 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5839
5840 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5841 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5842
5843 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5844 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5845
5846 if (VMMRZCallRing3IsEnabled(pVCpu))
5847 VMMR0LogFlushEnable(pVCpu);
5848
5849 return rc;
5850}
5851
5852
5853/**
5854 * Check per-VM and per-VCPU force flag actions that require us to go back to
5855 * ring-3 for one reason or another.
5856 *
5857 * @returns VBox status code (information status code included).
5858 * @retval VINF_SUCCESS if we don't have any actions that require going back to
5859 * ring-3.
5860 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5861 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5862 * interrupts)
5863 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5864 * all EMTs to be in ring-3.
5865 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5866 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5867 * to the EM loop.
5868 *
5869 * @param pVM Pointer to the VM.
5870 * @param pVCpu Pointer to the VMCPU.
5871 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5872 * out-of-sync. Make sure to update the required fields
5873 * before using them.
5874 */
5875static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5876{
5877 Assert(VMMRZCallRing3IsEnabled(pVCpu));
5878
5879 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5880 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5881 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5882 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5883 {
5884 /* We need the control registers now, make sure the guest-CPU context is updated. */
5885 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5886 AssertRCReturn(rc3, rc3);
5887
5888 /* Pending HM CR3 sync. */
5889 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5890 {
5891 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5892 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
5893 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5894 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5895 }
5896
5897 /* Pending HM PAE PDPEs. */
5898 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5899 {
5900 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5901 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5902 }
5903
5904 /* Pending PGM C3 sync. */
5905 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5906 {
5907 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
5908 VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5909 if (rc2 != VINF_SUCCESS)
5910 {
5911 AssertRC(rc2);
5912 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5913 return rc2;
5914 }
5915 }
5916
5917 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5918 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5919 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5920 {
5921 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5922 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5923 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5924 return rc2;
5925 }
5926
5927 /* Pending VM request packets, such as hardware interrupts. */
5928 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5929 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5930 {
5931 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5932 return VINF_EM_PENDING_REQUEST;
5933 }
5934
5935 /* Pending PGM pool flushes. */
5936 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5937 {
5938 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5939 return VINF_PGM_POOL_FLUSH_PENDING;
5940 }
5941
5942 /* Pending DMA requests. */
5943 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5944 {
5945 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5946 return VINF_EM_RAW_TO_R3;
5947 }
5948 }
5949
5950 return VINF_SUCCESS;
5951}
5952
5953
5954/**
5955 * Converts any TRPM trap into a pending HM event. This is typically used when
5956 * entering from ring-3 (not longjmp returns).
5957 *
5958 * @param pVCpu Pointer to the VMCPU.
5959 */
5960static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5961{
5962 Assert(TRPMHasTrap(pVCpu));
5963 Assert(!pVCpu->hm.s.Event.fPending);
5964
5965 uint8_t uVector;
5966 TRPMEVENT enmTrpmEvent;
5967 RTGCUINT uErrCode;
5968 RTGCUINTPTR GCPtrFaultAddress;
5969 uint8_t cbInstr;
5970
5971 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5972 AssertRC(rc);
5973
5974 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5975 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
5976 if (enmTrpmEvent == TRPM_TRAP)
5977 {
5978 switch (uVector)
5979 {
5980 case X86_XCPT_BP:
5981 case X86_XCPT_OF:
5982 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5983 break;
5984
5985 case X86_XCPT_PF:
5986 case X86_XCPT_DF:
5987 case X86_XCPT_TS:
5988 case X86_XCPT_NP:
5989 case X86_XCPT_SS:
5990 case X86_XCPT_GP:
5991 case X86_XCPT_AC:
5992 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5993 /* no break! */
5994 default:
5995 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5996 break;
5997 }
5998 }
5999 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6000 {
6001 if (uVector == X86_XCPT_NMI)
6002 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6003 else
6004 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6005 }
6006 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6007 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6008 else
6009 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6010
6011 rc = TRPMResetTrap(pVCpu);
6012 AssertRC(rc);
6013 Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6014 u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6015
6016 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6017 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6018}
6019
6020
6021/**
6022 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6023 * VT-x to execute any instruction.
6024 *
6025 * @param pvCpu Pointer to the VMCPU.
6026 */
6027static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6028{
6029 Assert(pVCpu->hm.s.Event.fPending);
6030
6031 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6032 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
6033 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
6034 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6035
6036 /* If a trap was already pending, we did something wrong! */
6037 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6038
6039 TRPMEVENT enmTrapType;
6040 switch (uVectorType)
6041 {
6042 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6043 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6044 enmTrapType = TRPM_HARDWARE_INT;
6045 break;
6046
6047 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6048 enmTrapType = TRPM_SOFTWARE_INT;
6049 break;
6050
6051 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6052 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6053 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6054 enmTrapType = TRPM_TRAP;
6055 break;
6056
6057 default:
6058 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6059 enmTrapType = TRPM_32BIT_HACK;
6060 break;
6061 }
6062
6063 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6064
6065 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6066 AssertRC(rc);
6067
6068 if (fErrorCodeValid)
6069 TRPMSetErrorCode(pVCpu, uErrorCode);
6070
6071 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6072 && uVector == X86_XCPT_PF)
6073 {
6074 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6075 }
6076 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6077 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6078 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6079 {
6080 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6081 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6082 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6083 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6084 }
6085 pVCpu->hm.s.Event.fPending = false;
6086}
6087
6088
6089/**
6090 * Does the necessary state syncing before returning to ring-3 for any reason
6091 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6092 *
6093 * @returns VBox status code.
6094 * @param pVM Pointer to the VM.
6095 * @param pVCpu Pointer to the VMCPU.
6096 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6097 * out-of-sync. Make sure to update the required fields
6098 * before using them.
6099 *
6100 * @remarks No-long-jmp zone!!!
6101 */
6102static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6103{
6104 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6105 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6106
6107 RTCPUID idCpu = RTMpCpuId();
6108 Log4Func(("HostCpuId=%u\n", idCpu));
6109
6110 /* Save the guest state if necessary. */
6111 if (pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6112 {
6113 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6114 AssertRCReturn(rc, rc);
6115 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6116 }
6117
6118 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6119 if (CPUMIsGuestFPUStateActive(pVCpu))
6120 {
6121 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6122 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6123 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
6124 }
6125
6126 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6127#ifdef VBOX_STRICT
6128 if (CPUMIsHyperDebugStateActive(pVCpu))
6129 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6130#endif
6131 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6132 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
6133 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6134 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6135
6136 /* Restore host-state bits that VT-x only restores partially. */
6137 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6138 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6139 {
6140 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6141 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6142 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6143 }
6144
6145 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6146 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6147 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6148 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6149 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6150 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6151 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6152 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6153
6154 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6155
6156 /** @todo This kinda defeats the purpose of having preemption hooks.
6157 * The problem is, deregistering the hooks should be moved to a place that
6158 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6159 * context.
6160 */
6161 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6162 {
6163 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6164 AssertRCReturn(rc, rc);
6165
6166 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6167 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6168 }
6169 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6170 NOREF(idCpu);
6171
6172 return VINF_SUCCESS;
6173}
6174
6175
6176/**
6177 * Leaves the VT-x session.
6178 *
6179 * @returns VBox status code.
6180 * @param pVM Pointer to the VM.
6181 * @param pVCpu Pointer to the VMCPU.
6182 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6183 * out-of-sync. Make sure to update the required fields
6184 * before using them.
6185 *
6186 * @remarks No-long-jmp zone!!!
6187 */
6188DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6189{
6190 HM_DISABLE_PREEMPT_IF_NEEDED();
6191 HMVMX_ASSERT_CPU_SAFE();
6192 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6193 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6194
6195 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6196 and done this from the VMXR0ThreadCtxCallback(). */
6197 if (!pVCpu->hm.s.fLeaveDone)
6198 {
6199 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6200 AssertRCReturn(rc2, rc2);
6201 pVCpu->hm.s.fLeaveDone = true;
6202 }
6203
6204 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6205 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6206 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6207 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6208 VMMR0ThreadCtxHooksDeregister(pVCpu);
6209
6210 /* Leave HM context. This takes care of local init (term). */
6211 int rc = HMR0LeaveCpu(pVCpu);
6212
6213 HM_RESTORE_PREEMPT_IF_NEEDED();
6214
6215 return rc;
6216}
6217
6218
6219/**
6220 * Does the necessary state syncing before doing a longjmp to ring-3.
6221 *
6222 * @returns VBox status code.
6223 * @param pVM Pointer to the VM.
6224 * @param pVCpu Pointer to the VMCPU.
6225 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6226 * out-of-sync. Make sure to update the required fields
6227 * before using them.
6228 *
6229 * @remarks No-long-jmp zone!!!
6230 */
6231DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6232{
6233 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6234}
6235
6236
6237/**
6238 * Take necessary actions before going back to ring-3.
6239 *
6240 * An action requires us to go back to ring-3. This function does the necessary
6241 * steps before we can safely return to ring-3. This is not the same as longjmps
6242 * to ring-3, this is voluntary and prepares the guest so it may continue
6243 * executing outside HM (recompiler/IEM).
6244 *
6245 * @returns VBox status code.
6246 * @param pVM Pointer to the VM.
6247 * @param pVCpu Pointer to the VMCPU.
6248 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6249 * out-of-sync. Make sure to update the required fields
6250 * before using them.
6251 * @param rcExit The reason for exiting to ring-3. Can be
6252 * VINF_VMM_UNKNOWN_RING3_CALL.
6253 */
6254static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6255{
6256 Assert(pVM);
6257 Assert(pVCpu);
6258 Assert(pMixedCtx);
6259 HMVMX_ASSERT_PREEMPT_SAFE();
6260
6261 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6262 {
6263 /* We've done what is required in hmR0VmxExitErrInvalidGuestState(). We're not going to continue guest execution... */
6264 return VINF_SUCCESS;
6265 }
6266 else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6267 {
6268 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6269 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6270 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6271 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6272 return VINF_SUCCESS;
6273 }
6274
6275 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6276 VMMRZCallRing3Disable(pVCpu);
6277 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6278
6279 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6280 if (pVCpu->hm.s.Event.fPending)
6281 {
6282 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6283 Assert(!pVCpu->hm.s.Event.fPending);
6284 }
6285
6286 /* Save guest state and restore host state bits. */
6287 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6288 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6289
6290 /* Sync recompiler state. */
6291 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6292 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6293 | CPUM_CHANGED_LDTR
6294 | CPUM_CHANGED_GDTR
6295 | CPUM_CHANGED_IDTR
6296 | CPUM_CHANGED_TR
6297 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6298 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6299 if ( pVM->hm.s.fNestedPaging
6300 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6301 {
6302 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6303 }
6304
6305 /*
6306 * Clear the X86_EFL_TF if necessary.
6307 */
6308 if (pVCpu->hm.s.fClearTrapFlag)
6309 {
6310 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6311 pMixedCtx->eflags.Bits.u1TF = 0;
6312 pVCpu->hm.s.fClearTrapFlag = false;
6313 }
6314 /** @todo there seems to be issues with the resume flag when the monitor trap
6315 * flag is pending without being used. Seen early in bios init when
6316 * accessing APIC page in prot mode. */
6317
6318 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6319 if (rcExit != VINF_EM_RAW_INTERRUPT)
6320 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
6321
6322 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6323
6324 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6325 VMMRZCallRing3RemoveNotification(pVCpu);
6326 VMMRZCallRing3Enable(pVCpu);
6327
6328 return rc;
6329}
6330
6331
6332/**
6333 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6334 * longjump to ring-3 and possibly get preempted.
6335 *
6336 * @returns VBox status code.
6337 * @param pVCpu Pointer to the VMCPU.
6338 * @param enmOperation The operation causing the ring-3 longjump.
6339 * @param pvUser Opaque pointer to the guest-CPU context. The data
6340 * may be out-of-sync. Make sure to update the required
6341 * fields before using them.
6342 *
6343 * @remarks Must never be called with @a enmOperation ==
6344 * VMMCALLRING3_VM_R0_ASSERTION. We can't assert it here because if it
6345 * it -does- get called with VMMCALLRING3_VM_R0_ASSERTION, we'll end up
6346 * with an infinite recursion.
6347 */
6348DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6349{
6350 /* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6351 Assert(pVCpu);
6352 Assert(pvUser);
6353 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6354 HMVMX_ASSERT_PREEMPT_SAFE();
6355
6356 VMMRZCallRing3Disable(pVCpu);
6357 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6358
6359 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6360 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6361 AssertRCReturn(rc, rc);
6362
6363 VMMRZCallRing3Enable(pVCpu);
6364 return VINF_SUCCESS;
6365}
6366
6367
6368/**
6369 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6370 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6371 *
6372 * @param pVCpu Pointer to the VMCPU.
6373 */
6374DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6375{
6376 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6377 {
6378 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6379 {
6380 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6381 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6382 AssertRC(rc);
6383 }
6384 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6385}
6386
6387
6388/**
6389 * Evaluates the event to be delivered to the guest and sets it as the pending
6390 * event.
6391 *
6392 * @param pVCpu Pointer to the VMCPU.
6393 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6394 * out-of-sync. Make sure to update the required fields
6395 * before using them.
6396 */
6397static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6398{
6399 Assert(!pVCpu->hm.s.Event.fPending);
6400
6401 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6402 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6403 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6404 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6405
6406 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6407 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6408 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6409 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6410 Assert(!TRPMHasTrap(pVCpu));
6411
6412 /** @todo SMI. SMIs take priority over NMIs. */
6413 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6414 {
6415 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6416 if ( !fBlockMovSS
6417 && !fBlockSti)
6418 {
6419 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6420 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6421 uint32_t u32IntrInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6422 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6423
6424 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6425 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6426 }
6427 else
6428 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6429 }
6430 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6431 && !pVCpu->hm.s.fSingleInstruction)
6432 {
6433 /*
6434 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6435 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6436 * evaluated here and not set as pending, solely based on the force-flags.
6437 */
6438 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6439 AssertRC(rc);
6440 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6441 if ( !fBlockInt
6442 && !fBlockSti
6443 && !fBlockMovSS)
6444 {
6445 uint8_t u8Interrupt;
6446 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6447 if (RT_SUCCESS(rc))
6448 {
6449 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6450 uint32_t u32IntrInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6451 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6452
6453 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6454 }
6455 else
6456 {
6457 /** @todo Does this actually happen? If not turn it into an assertion. */
6458 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6459 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6460 }
6461 }
6462 else
6463 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6464 }
6465}
6466
6467
6468/**
6469 * Injects any pending events into the guest if the guest is in a state to
6470 * receive them.
6471 *
6472 * @returns VBox status code (informational status codes included).
6473 * @param pVCpu Pointer to the VMCPU.
6474 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6475 * out-of-sync. Make sure to update the required fields
6476 * before using them.
6477 *
6478 * @remarks No-long-jump zone!!!
6479 */
6480static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6481{
6482 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6483 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6484 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6485 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6486
6487 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6488 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6489 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6490 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6491 Assert(!TRPMHasTrap(pVCpu));
6492
6493 int rc = VINF_SUCCESS;
6494 if (pVCpu->hm.s.Event.fPending)
6495 {
6496#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6497 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6498 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6499 {
6500 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6501 AssertRCReturn(rc, rc);
6502 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6503 Assert(!fBlockInt);
6504 Assert(!fBlockSti);
6505 Assert(!fBlockMovSS);
6506 }
6507 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6508 {
6509 Assert(!fBlockSti);
6510 Assert(!fBlockMovSS);
6511 }
6512#endif
6513 Log4(("Injecting pending event vcpu[%RU32] u64IntrInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo));
6514 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6515 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6516 AssertRCReturn(rc, rc);
6517
6518 pVCpu->hm.s.Event.fPending = false;
6519
6520 /* Update the interruptibility-state as it could have been changed by
6521 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6522 fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6523 fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6524
6525#ifdef VBOX_WITH_STATISTICS
6526 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6527 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6528 else
6529 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6530#endif
6531 }
6532
6533 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6534 int rc2 = VINF_SUCCESS;
6535 if ( fBlockSti
6536 || fBlockMovSS)
6537 {
6538 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6539 {
6540 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6541 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6542 {
6543 /*
6544 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6545 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6546 * See Intel spec. 27.3.4 "Saving Non-Register State".
6547 */
6548 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6549 AssertRCReturn(rc, rc);
6550 }
6551 }
6552 else
6553 {
6554 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6555 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6556 uIntrState = 0;
6557 }
6558 }
6559
6560 /*
6561 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6562 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6563 */
6564 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6565 AssertRC(rc2);
6566
6567 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6568 return rc;
6569}
6570
6571
6572/**
6573 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6574 *
6575 * @param pVCpu Pointer to the VMCPU.
6576 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6577 * out-of-sync. Make sure to update the required fields
6578 * before using them.
6579 */
6580DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6581{
6582 uint32_t u32IntrInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6583 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6584}
6585
6586
6587/**
6588 * Injects a double-fault (#DF) exception into the VM.
6589 *
6590 * @returns VBox status code (informational status code included).
6591 * @param pVCpu Pointer to the VMCPU.
6592 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6593 * out-of-sync. Make sure to update the required fields
6594 * before using them.
6595 */
6596DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6597{
6598 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6599 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6600 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6601 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6602 puIntrState);
6603}
6604
6605
6606/**
6607 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6608 *
6609 * @param pVCpu Pointer to the VMCPU.
6610 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6611 * out-of-sync. Make sure to update the required fields
6612 * before using them.
6613 */
6614DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6615{
6616 uint32_t u32IntrInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6617 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6618 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6619}
6620
6621
6622/**
6623 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6624 *
6625 * @param pVCpu Pointer to the VMCPU.
6626 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6627 * out-of-sync. Make sure to update the required fields
6628 * before using them.
6629 * @param cbInstr The value of RIP that is to be pushed on the guest
6630 * stack.
6631 */
6632DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6633{
6634 uint32_t u32IntrInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6635 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6636 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6637}
6638
6639
6640/**
6641 * Injects a general-protection (#GP) fault into the VM.
6642 *
6643 * @returns VBox status code (informational status code included).
6644 * @param pVCpu Pointer to the VMCPU.
6645 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6646 * out-of-sync. Make sure to update the required fields
6647 * before using them.
6648 * @param u32ErrorCode The error code associated with the #GP.
6649 */
6650DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6651 uint32_t *puIntrState)
6652{
6653 uint32_t u32IntrInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6654 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6655 if (fErrorCodeValid)
6656 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6657 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6658 puIntrState);
6659}
6660
6661
6662/**
6663 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6664 *
6665 * @param pVCpu Pointer to the VMCPU.
6666 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6667 * out-of-sync. Make sure to update the required fields
6668 * before using them.
6669 * @param uVector The software interrupt vector number.
6670 * @param cbInstr The value of RIP that is to be pushed on the guest
6671 * stack.
6672 */
6673DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6674{
6675 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6676 if ( uVector == X86_XCPT_BP
6677 || uVector == X86_XCPT_OF)
6678 {
6679 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6680 }
6681 else
6682 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6683 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6684}
6685
6686
6687/**
6688 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6689 * stack.
6690 *
6691 * @returns VBox status code (information status code included).
6692 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6693 * @param pVM Pointer to the VM.
6694 * @param pMixedCtx Pointer to the guest-CPU context.
6695 * @param uValue The value to push to the guest stack.
6696 */
6697DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6698{
6699 /*
6700 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6701 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6702 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6703 */
6704 if (pMixedCtx->sp == 1)
6705 return VINF_EM_RESET;
6706 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6707 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6708 AssertRCReturn(rc, rc);
6709 return rc;
6710}
6711
6712
6713/**
6714 * Injects an event into the guest upon VM-entry by updating the relevant fields
6715 * in the VM-entry area in the VMCS.
6716 *
6717 * @returns VBox status code (informational error codes included).
6718 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6719 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6720 *
6721 * @param pVCpu Pointer to the VMCPU.
6722 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6723 * be out-of-sync. Make sure to update the required
6724 * fields before using them.
6725 * @param u64IntrInfo The VM-entry interruption-information field.
6726 * @param cbInstr The VM-entry instruction length in bytes (for
6727 * software interrupts, exceptions and privileged
6728 * software exceptions).
6729 * @param u32ErrCode The VM-entry exception error code.
6730 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6731 * @param puIntrState Pointer to the current guest interruptibility-state.
6732 * This interruptibility-state will be updated if
6733 * necessary. This cannot not be NULL.
6734 *
6735 * @remarks No-long-jump zone!!!
6736 * @remarks Requires CR0!
6737 */
6738static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6739 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6740{
6741 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6742 AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6743 Assert(puIntrState);
6744 uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6745
6746 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6747 const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6748
6749#ifdef VBOX_STRICT
6750 /* Validate the error-code-valid bit for hardware exceptions. */
6751 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6752 {
6753 switch (uVector)
6754 {
6755 case X86_XCPT_PF:
6756 case X86_XCPT_DF:
6757 case X86_XCPT_TS:
6758 case X86_XCPT_NP:
6759 case X86_XCPT_SS:
6760 case X86_XCPT_GP:
6761 case X86_XCPT_AC:
6762 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6763 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6764 /* fallthru */
6765 default:
6766 break;
6767 }
6768 }
6769#endif
6770
6771 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
6772 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6773 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6774
6775 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6776
6777 /* We require CR0 to check if the guest is in real-mode. */
6778 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6779 AssertRCReturn(rc, rc);
6780
6781 /*
6782 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6783 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6784 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6785 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6786 */
6787 if (CPUMIsGuestInRealModeEx(pMixedCtx))
6788 {
6789 PVM pVM = pVCpu->CTX_SUFF(pVM);
6790 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6791 {
6792 Assert(PDMVmmDevHeapIsEnabled(pVM));
6793 Assert(pVM->hm.s.vmx.pRealModeTSS);
6794
6795 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6796 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6797 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6798 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6799 AssertRCReturn(rc, rc);
6800 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6801
6802 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6803 const size_t cbIdtEntry = 4;
6804 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6805 {
6806 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6807 if (uVector == X86_XCPT_DF)
6808 return VINF_EM_RESET;
6809 else if (uVector == X86_XCPT_GP)
6810 {
6811 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6812 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6813 }
6814
6815 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6816 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6817 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
6818 }
6819
6820 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6821 uint16_t uGuestIp = pMixedCtx->ip;
6822 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6823 {
6824 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
6825 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
6826 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6827 }
6828 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6829 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6830
6831 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6832 uint16_t offIdtEntry = 0;
6833 RTSEL selIdtEntry = 0;
6834 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6835 rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6836 rc |= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6837 AssertRCReturn(rc, rc);
6838
6839 /* Construct the stack frame for the interrupt/exception handler. */
6840 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6841 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6842 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6843 AssertRCReturn(rc, rc);
6844
6845 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
6846 if (rc == VINF_SUCCESS)
6847 {
6848 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
6849 pMixedCtx->rip = offIdtEntry;
6850 pMixedCtx->cs.Sel = selIdtEntry;
6851 pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6852 if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6853 && uVector == X86_XCPT_PF)
6854 {
6855 pMixedCtx->cr2 = GCPtrFaultAddress;
6856 }
6857
6858 /* If any other guest-state bits are changed here, make sure to update
6859 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6860 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS
6861 | HM_CHANGED_GUEST_RIP
6862 | HM_CHANGED_GUEST_RFLAGS
6863 | HM_CHANGED_GUEST_RSP;
6864
6865 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6866 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6867 {
6868 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6869 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6870 Log4(("Clearing inhibition due to STI.\n"));
6871 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6872 }
6873 Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6874 }
6875 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6876 return rc;
6877 }
6878 else
6879 {
6880 /*
6881 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6882 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6883 */
6884 u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6885 }
6886 }
6887
6888 /* Validate. */
6889 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6890 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6891 Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6892
6893 /* Inject. */
6894 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6895 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6896 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6897 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6898
6899 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6900 && uVector == X86_XCPT_PF)
6901 {
6902 pMixedCtx->cr2 = GCPtrFaultAddress;
6903 }
6904
6905 Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6906 u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6907
6908 AssertRCReturn(rc, rc);
6909 return rc;
6910}
6911
6912
6913/**
6914 * Enters the VT-x session.
6915 *
6916 * @returns VBox status code.
6917 * @param pVM Pointer to the VM.
6918 * @param pVCpu Pointer to the VMCPU.
6919 * @param pCpu Pointer to the CPU info struct.
6920 */
6921VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
6922{
6923 AssertPtr(pVM);
6924 AssertPtr(pVCpu);
6925 Assert(pVM->hm.s.vmx.fSupported);
6926 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6927 NOREF(pCpu);
6928
6929 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6930 Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
6931
6932#ifdef VBOX_STRICT
6933 /* Make sure we're in VMX root mode. */
6934 RTCCUINTREG u32HostCR4 = ASMGetCR4();
6935 if (!(u32HostCR4 & X86_CR4_VMXE))
6936 {
6937 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6938 return VERR_VMX_X86_CR4_VMXE_CLEARED;
6939 }
6940#endif
6941
6942 /*
6943 * Load the VCPU's VMCS as the current (and active) one.
6944 */
6945 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
6946 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6947 if (RT_FAILURE(rc))
6948 return rc;
6949
6950 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
6951 pVCpu->hm.s.fLeaveDone = false;
6952 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
6953
6954 return VINF_SUCCESS;
6955}
6956
6957
6958/**
6959 * The thread-context callback (only on platforms which support it).
6960 *
6961 * @param enmEvent The thread-context event.
6962 * @param pVCpu Pointer to the VMCPU.
6963 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
6964 * @thread EMT.
6965 */
6966VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
6967{
6968 switch (enmEvent)
6969 {
6970 case RTTHREADCTXEVENT_PREEMPTING:
6971 {
6972 /** @todo Stats. */
6973 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6974 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
6975 VMCPU_ASSERT_EMT(pVCpu);
6976
6977 PVM pVM = pVCpu->CTX_SUFF(pVM);
6978 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
6979
6980 /* No longjmps (logger flushes, locks) in this fragile context. */
6981 VMMRZCallRing3Disable(pVCpu);
6982 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
6983
6984 /* Save the guest-state, restore host-state (FPU, debug etc.). */
6985 if (!pVCpu->hm.s.fLeaveDone)
6986 {
6987 hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6988 pVCpu->hm.s.fLeaveDone = true;
6989 }
6990
6991 /* Leave HM context, takes care of local init (term). */
6992 int rc = HMR0LeaveCpu(pVCpu);
6993 AssertRC(rc); NOREF(rc);
6994
6995 /* Restore longjmp state. */
6996 VMMRZCallRing3Enable(pVCpu);
6997 break;
6998 }
6999
7000 case RTTHREADCTXEVENT_RESUMED:
7001 {
7002 /** @todo Stats. */
7003 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7004 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7005 VMCPU_ASSERT_EMT(pVCpu);
7006
7007 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7008 VMMRZCallRing3Disable(pVCpu);
7009 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7010
7011 /* Initialize the bare minimum state required for HM. This takes care of
7012 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7013 int rc = HMR0EnterCpu(pVCpu);
7014 AssertRC(rc);
7015 Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7016
7017 /* Load the active VMCS as the current one. */
7018 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7019 {
7020 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7021 AssertRC(rc); NOREF(rc);
7022 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7023 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7024 }
7025 pVCpu->hm.s.fLeaveDone = false;
7026 VMMRZCallRing3Enable(pVCpu);
7027 break;
7028 }
7029
7030 default:
7031 break;
7032 }
7033}
7034
7035
7036/**
7037 * Saves the host state in the VMCS host-state.
7038 * Sets up the VM-exit MSR-load area.
7039 *
7040 * The CPU state will be loaded from these fields on every successful VM-exit.
7041 *
7042 * @returns VBox status code.
7043 * @param pVM Pointer to the VM.
7044 * @param pVCpu Pointer to the VMCPU.
7045 *
7046 * @remarks No-long-jump zone!!!
7047 */
7048static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7049{
7050 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7051
7052 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
7053 return VINF_SUCCESS;
7054
7055 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7056 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7057
7058 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7059 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7060
7061 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7062 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7063
7064 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
7065 return rc;
7066}
7067
7068
7069/**
7070 * Saves the host state in the VMCS host-state.
7071 *
7072 * @returns VBox status code.
7073 * @param pVM Pointer to the VM.
7074 * @param pVCpu Pointer to the VMCPU.
7075 *
7076 * @remarks No-long-jump zone!!!
7077 */
7078VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7079{
7080 AssertPtr(pVM);
7081 AssertPtr(pVCpu);
7082
7083 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7084
7085 /* When thread-context hooks are available, this is done later (when preemption/interrupts are disabled). */
7086 if (!VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7087 {
7088 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7089 return hmR0VmxSaveHostState(pVM, pVCpu);
7090 }
7091 return VINF_SUCCESS;
7092}
7093
7094
7095/**
7096 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7097 * loaded from these fields on every successful VM-entry.
7098 *
7099 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7100 * Sets up the VM-entry controls.
7101 * Sets up the appropriate VMX non-root function to execute guest code based on
7102 * the guest CPU mode.
7103 *
7104 * @returns VBox status code.
7105 * @param pVM Pointer to the VM.
7106 * @param pVCpu Pointer to the VMCPU.
7107 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7108 * out-of-sync. Make sure to update the required fields
7109 * before using them.
7110 *
7111 * @remarks No-long-jump zone!!!
7112 */
7113static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7114{
7115 AssertPtr(pVM);
7116 AssertPtr(pVCpu);
7117 AssertPtr(pMixedCtx);
7118 HMVMX_ASSERT_PREEMPT_SAFE();
7119
7120#ifdef LOG_ENABLED
7121 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7122 * probably not initialized yet? Anyway this will do for now.
7123 *
7124 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7125 * interface and disable ring-3 calls when thread-context hooks are not
7126 * available. */
7127 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7128 VMMR0LogFlushDisable(pVCpu);
7129#endif
7130
7131 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7132
7133 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7134
7135 /* Determine real-on-v86 mode. */
7136 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7137 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7138 && CPUMIsGuestInRealModeEx(pMixedCtx))
7139 {
7140 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7141 }
7142
7143 /*
7144 * Load the guest-state into the VMCS.
7145 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7146 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7147 */
7148 int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7149 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7150
7151 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7152 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7153
7154 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7155 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7156
7157 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7158 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7159
7160 /* Assumes CR0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7161 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7162 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7163
7164 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7165 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7166
7167 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7168 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7169
7170 /*
7171 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7172 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7173 */
7174 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7175 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7176
7177 rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7178 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7179
7180 /* Clear any unused and reserved bits. */
7181 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
7182
7183#ifdef LOG_ENABLED
7184 /* Only reenable log-flushing if the caller has it enabled. */
7185 if (!fCallerDisabledLogFlush)
7186 VMMR0LogFlushEnable(pVCpu);
7187#endif
7188
7189 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7190 return rc;
7191}
7192
7193
7194/**
7195 * Loads the state shared between the host and guest into the VMCS.
7196 *
7197 * @param pVM Pointer to the VM.
7198 * @param pVCpu Pointer to the VMCPU.
7199 * @param pCtx Pointer to the guest-CPU context.
7200 *
7201 * @remarks No-long-jump zone!!!
7202 */
7203static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7204{
7205 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7206 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7207
7208 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
7209 {
7210 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7211 AssertRC(rc);
7212 }
7213
7214 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
7215 {
7216 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7217 AssertRC(rc);
7218
7219 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7220 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
7221 {
7222 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7223 AssertRC(rc);
7224 }
7225 }
7226
7227 AssertMsg(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE), ("fContextUseFlags=%#x\n",
7228 pVCpu->hm.s.fContextUseFlags));
7229}
7230
7231
7232/**
7233 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7234 *
7235 * @param pVM Pointer to the VM.
7236 * @param pVCpu Pointer to the VMCPU.
7237 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7238 * out-of-sync. Make sure to update the required fields
7239 * before using them.
7240 */
7241DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7242{
7243 HMVMX_ASSERT_PREEMPT_SAFE();
7244
7245 Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7246#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7247 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
7248#endif
7249
7250 if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7251 {
7252 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7253 AssertRC(rc);
7254 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7255 }
7256 else if (pVCpu->hm.s.fContextUseFlags)
7257 {
7258 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7259 AssertRC(rc);
7260 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7261 }
7262
7263 /* All the guest state bits should be loaded except maybe the host context and shared host/guest bits. */
7264 AssertMsg( !(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST)
7265 || !(pVCpu->hm.s.fContextUseFlags & ~(HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE)),
7266 ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7267
7268#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7269 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7270 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7271 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7272#endif
7273}
7274
7275
7276/**
7277 * Does the preparations before executing guest code in VT-x.
7278 *
7279 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7280 * recompiler. We must be cautious what we do here regarding committing
7281 * guest-state information into the VMCS assuming we assuredly execute the
7282 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
7283 * and clearing the common-state (TRPM/forceflags), we must undo those changes
7284 * so that the recompiler can (and should) use them when it resumes guest
7285 * execution. Otherwise such operations must be done when we can no longer
7286 * exit to ring-3.
7287 *
7288 * @returns Strict VBox status code.
7289 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
7290 * have been disabled.
7291 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
7292 * double-fault into the guest.
7293 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7294 *
7295 * @param pVM Pointer to the VM.
7296 * @param pVCpu Pointer to the VMCPU.
7297 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7298 * out-of-sync. Make sure to update the required fields
7299 * before using them.
7300 * @param pVmxTransient Pointer to the VMX transient structure.
7301 *
7302 * @remarks Called with preemption disabled. In the VINF_SUCCESS return case
7303 * interrupts will be disabled.
7304 */
7305static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7306{
7307 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7308
7309#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7310 PGMRZDynMapFlushAutoSet(pVCpu);
7311#endif
7312
7313 /* Check force flag actions that might require us to go back to ring-3. */
7314 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7315 if (rc != VINF_SUCCESS)
7316 return rc;
7317
7318#ifndef IEM_VERIFICATION_MODE_FULL
7319 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7320 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7321 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7322 {
7323 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7324 RTGCPHYS GCPhysApicBase;
7325 GCPhysApicBase = pMixedCtx->msrApicBase;
7326 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7327
7328 /* Unalias any existing mapping. */
7329 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7330 AssertRCReturn(rc, rc);
7331
7332 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7333 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7334 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7335 AssertRCReturn(rc, rc);
7336
7337 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7338 }
7339#endif /* !IEM_VERIFICATION_MODE_FULL */
7340
7341 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7342 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7343
7344 /*
7345 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7346 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7347 */
7348 if (TRPMHasTrap(pVCpu))
7349 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7350 else if (!pVCpu->hm.s.Event.fPending)
7351 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7352
7353 /*
7354 * No longjmps to ring-3 from this point on!!!
7355 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7356 * This also disables flushing of the R0-logger instance (if any).
7357 */
7358 VMMRZCallRing3Disable(pVCpu);
7359
7360 /*
7361 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7362 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7363 *
7364 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7365 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7366 *
7367 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7368 * executing guest code.
7369 */
7370 pVmxTransient->uEflags = ASMIntDisableFlags();
7371 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7372 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7373 {
7374 ASMSetFlags(pVmxTransient->uEflags);
7375 VMMRZCallRing3Enable(pVCpu);
7376 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7377 return VINF_EM_RAW_TO_R3;
7378 }
7379 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7380 {
7381 ASMSetFlags(pVmxTransient->uEflags);
7382 VMMRZCallRing3Enable(pVCpu);
7383 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7384 return VINF_EM_RAW_INTERRUPT;
7385 }
7386
7387 /*
7388 * Event injection might result in triple-faulting the VM (real-on-v86 case), which is why it's
7389 * done here and not in hmR0VmxPreRunGuestCommitted() which doesn't expect failures.
7390 */
7391 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7392 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7393 {
7394 ASMSetFlags(pVmxTransient->uEflags);
7395 VMMRZCallRing3Enable(pVCpu);
7396 return rc;
7397 }
7398
7399 return VINF_SUCCESS;
7400}
7401
7402
7403/**
7404 * Prepares to run guest code in VT-x and we've committed to doing so. This
7405 * means there is no backing out to ring-3 or anywhere else at this
7406 * point.
7407 *
7408 * @param pVM Pointer to the VM.
7409 * @param pVCpu Pointer to the VMCPU.
7410 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7411 * out-of-sync. Make sure to update the required fields
7412 * before using them.
7413 * @param pVmxTransient Pointer to the VMX transient structure.
7414 *
7415 * @remarks Called with preemption disabled.
7416 * @remarks No-long-jump zone!!!
7417 */
7418static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7419{
7420 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7421 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7422 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7423
7424 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7425 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7426
7427 /*
7428 * Load the host state bits as we may've been preempted (only happens when
7429 * thread-context hooks are used).
7430 */
7431 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
7432 {
7433 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7434 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7435 AssertRC(rc);
7436 }
7437 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7438
7439 /*
7440 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7441 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7442 * Reload only the necessary state, the assertion will catch if other parts of the code
7443 * change.
7444 */
7445 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7446 {
7447 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7448 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7449 }
7450
7451 /* Load the state shared between host and guest (FPU, debug). */
7452 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE)
7453 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7454 AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7455
7456 /*
7457 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7458 */
7459 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7460 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7461
7462 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
7463 RTCPUID idCurrentCpu = pCpu->idCpu;
7464 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7465 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
7466 {
7467 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7468 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7469 }
7470
7471 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7472 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
7473
7474 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7475 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7476
7477 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7478
7479 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7480 to start executing. */
7481
7482#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7483 /*
7484 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7485 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7486 */
7487 if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7488 && !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7489 {
7490 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7491 uint64_t u64HostTscAux = 0;
7492 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7493 AssertRC(rc2);
7494 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7495 }
7496#endif
7497}
7498
7499
7500/**
7501 * Performs some essential restoration of state after running guest code in
7502 * VT-x.
7503 *
7504 * @param pVM Pointer to the VM.
7505 * @param pVCpu Pointer to the VMCPU.
7506 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7507 * out-of-sync. Make sure to update the required fields
7508 * before using them.
7509 * @param pVmxTransient Pointer to the VMX transient structure.
7510 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7511 *
7512 * @remarks Called with interrupts disabled, and returns with interrups enabled!
7513 *
7514 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7515 * unconditionally when it is safe to do so.
7516 */
7517static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7518{
7519 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7520
7521 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7522 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7523 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7524 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7525 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7526
7527 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7528 {
7529#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7530 /* Restore host's TSC_AUX. */
7531 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7532 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7533#endif
7534 /** @todo Find a way to fix hardcoding a guestimate. */
7535 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7536 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7537 }
7538
7539 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7540 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7541 Assert(!(ASMGetFlags() & X86_EFL_IF));
7542 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7543
7544 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7545 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7546 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7547 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7548
7549 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7550 uint32_t uExitReason;
7551 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7552 rc |= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7553 AssertRC(rc);
7554 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7555 pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7556
7557 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7558 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7559 {
7560 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7561 pVmxTransient->fVMEntryFailed));
7562 return;
7563 }
7564
7565 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7566 {
7567 /* Update the guest interruptibility-state from the VMCS. */
7568 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7569#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7570 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7571 AssertRC(rc);
7572#endif
7573 /*
7574 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7575 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7576 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7577 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7578 */
7579 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7580 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7581 {
7582 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7583 AssertRC(rc);
7584 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
7585 }
7586 }
7587}
7588
7589
7590
7591/**
7592 * Runs the guest code using VT-x the normal way.
7593 *
7594 * @returns VBox status code.
7595 * @param pVM Pointer to the VM.
7596 * @param pVCpu Pointer to the VMCPU.
7597 * @param pCtx Pointer to the guest-CPU context.
7598 *
7599 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
7600 * @remarks Called with preemption disabled.
7601 */
7602static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7603{
7604 VMXTRANSIENT VmxTransient;
7605 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7606 int rc = VERR_INTERNAL_ERROR_5;
7607 uint32_t cLoops = 0;
7608
7609 for (;; cLoops++)
7610 {
7611 Assert(!HMR0SuspendPending());
7612 HMVMX_ASSERT_CPU_SAFE();
7613
7614 /* Preparatory work for running guest code, this may force us to return
7615 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7616 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7617 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7618 if (rc != VINF_SUCCESS)
7619 break;
7620
7621 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7622 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7623 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7624
7625 /* Restore any residual host-state and save any bits shared between host
7626 and guest into the guest-CPU state. Re-enables interrupts! */
7627 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7628
7629 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7630 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7631 {
7632 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7633 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7634 return rc;
7635 }
7636
7637 /* Handle the VM-exit. */
7638 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7639 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7640 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7641 HMVMX_START_EXIT_DISPATCH_PROF();
7642#ifdef HMVMX_USE_FUNCTION_TABLE
7643 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7644#else
7645 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7646#endif
7647 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7648 if (rc != VINF_SUCCESS)
7649 break;
7650 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7651 {
7652 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7653 rc = VINF_EM_RAW_INTERRUPT;
7654 break;
7655 }
7656 }
7657
7658 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7659 return rc;
7660}
7661
7662
7663/**
7664 * Single steps guest code using VT-x.
7665 *
7666 * @returns VBox status code.
7667 * @param pVM Pointer to the VM.
7668 * @param pVCpu Pointer to the VMCPU.
7669 * @param pCtx Pointer to the guest-CPU context.
7670 *
7671 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7672 * @remarks Called with preemption disabled.
7673 */
7674static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7675{
7676 VMXTRANSIENT VmxTransient;
7677 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7678 int rc = VERR_INTERNAL_ERROR_5;
7679 uint32_t cLoops = 0;
7680 uint16_t uCsStart = pCtx->cs.Sel;
7681 uint64_t uRipStart = pCtx->rip;
7682
7683 for (;; cLoops++)
7684 {
7685 Assert(!HMR0SuspendPending());
7686 HMVMX_ASSERT_CPU_SAFE();
7687
7688 /* Preparatory work for running guest code, this may force us to return
7689 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
7690 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7691 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7692 if (rc != VINF_SUCCESS)
7693 break;
7694
7695 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7696 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7697 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7698
7699 /* Restore any residual host-state and save any bits shared between host
7700 and guest into the guest-CPU state. Re-enables interrupts! */
7701 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7702
7703 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7704 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7705 {
7706 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7707 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7708 return rc;
7709 }
7710
7711 /* Handle the VM-exit. */
7712 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7713 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7714 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7715 HMVMX_START_EXIT_DISPATCH_PROF();
7716#ifdef HMVMX_USE_FUNCTION_TABLE
7717 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7718#else
7719 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7720#endif
7721 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7722 if (rc != VINF_SUCCESS)
7723 break;
7724 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7725 {
7726 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7727 rc = VINF_EM_RAW_INTERRUPT;
7728 break;
7729 }
7730
7731 /*
7732 * Did the RIP change, if so, consider it a single step.
7733 * Otherwise, make sure one of the TFs gets set.
7734 */
7735 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7736 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7737 AssertRCReturn(rc2, rc2);
7738 if ( pCtx->rip != uRipStart
7739 || pCtx->cs.Sel != uCsStart)
7740 {
7741 rc = VINF_EM_DBG_STEPPED;
7742 break;
7743 }
7744 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
7745 }
7746
7747 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7748 return rc;
7749}
7750
7751
7752/**
7753 * Runs the guest code using VT-x.
7754 *
7755 * @returns VBox status code.
7756 * @param pVM Pointer to the VM.
7757 * @param pVCpu Pointer to the VMCPU.
7758 * @param pCtx Pointer to the guest-CPU context.
7759 *
7760 * @remarks Called with preemption disabled.
7761 */
7762VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7763{
7764 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7765 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
7766 HMVMX_ASSERT_PREEMPT_SAFE();
7767
7768 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
7769
7770 int rc;
7771 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7772 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7773 else
7774 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7775
7776 if (rc == VERR_EM_INTERPRETER)
7777 rc = VINF_EM_RAW_EMULATE_INSTR;
7778 else if (rc == VINF_EM_RESET)
7779 rc = VINF_EM_TRIPLE_FAULT;
7780
7781 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7782 if (RT_FAILURE(rc2))
7783 {
7784 pVCpu->hm.s.u32HMError = rc;
7785 rc = rc2;
7786 }
7787 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
7788 return rc;
7789}
7790
7791
7792#ifndef HMVMX_USE_FUNCTION_TABLE
7793DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7794{
7795 int rc;
7796 switch (rcReason)
7797 {
7798 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7799 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7800 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7801 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7802 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7803 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7804 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7805 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7806 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7807 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7808 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7809 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7810 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7811 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7812 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7813 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7814 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7815 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7816 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7817 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7818 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7819 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7820 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7821 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7822 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7823 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7824 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7825 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7826 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7827 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7828 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7829 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7830 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7831
7832 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7833 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7834 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7835 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7836 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7837 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7838 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7839 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7840 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7841
7842 case VMX_EXIT_VMCALL:
7843 case VMX_EXIT_VMCLEAR:
7844 case VMX_EXIT_VMLAUNCH:
7845 case VMX_EXIT_VMPTRLD:
7846 case VMX_EXIT_VMPTRST:
7847 case VMX_EXIT_VMREAD:
7848 case VMX_EXIT_VMRESUME:
7849 case VMX_EXIT_VMWRITE:
7850 case VMX_EXIT_VMXOFF:
7851 case VMX_EXIT_VMXON:
7852 case VMX_EXIT_INVEPT:
7853 case VMX_EXIT_INVVPID:
7854 case VMX_EXIT_VMFUNC:
7855 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7856 break;
7857 default:
7858 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7859 break;
7860 }
7861 return rc;
7862}
7863#endif
7864
7865#ifdef DEBUG
7866/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7867# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7868 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7869
7870# define HMVMX_ASSERT_PREEMPT_CPUID() \
7871 do \
7872 { \
7873 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7874 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7875 } while (0)
7876
7877# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7878 do { \
7879 AssertPtr(pVCpu); \
7880 AssertPtr(pMixedCtx); \
7881 AssertPtr(pVmxTransient); \
7882 Assert(pVmxTransient->fVMEntryFailed == false); \
7883 Assert(ASMIntAreEnabled()); \
7884 HMVMX_ASSERT_PREEMPT_SAFE(); \
7885 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7886 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7887 HMVMX_ASSERT_PREEMPT_SAFE(); \
7888 if (VMMR0IsLogFlushDisabled(pVCpu)) \
7889 HMVMX_ASSERT_PREEMPT_CPUID(); \
7890 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7891 } while (0)
7892
7893# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7894 do { \
7895 Log4Func(("\n")); \
7896 } while(0)
7897#else /* Release builds */
7898# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7899# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7900#endif
7901
7902
7903/**
7904 * Advances the guest RIP after reading it from the VMCS.
7905 *
7906 * @returns VBox status code.
7907 * @param pVCpu Pointer to the VMCPU.
7908 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7909 * out-of-sync. Make sure to update the required fields
7910 * before using them.
7911 * @param pVmxTransient Pointer to the VMX transient structure.
7912 *
7913 * @remarks No-long-jump zone!!!
7914 */
7915DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7916{
7917 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7918 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7919 AssertRCReturn(rc, rc);
7920
7921 pMixedCtx->rip += pVmxTransient->cbInstr;
7922 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
7923 return rc;
7924}
7925
7926
7927/**
7928 * Tries to determine what part of the guest-state VT-x has deemed as invalid
7929 * and update error record fields accordingly.
7930 *
7931 * @return VMX_IGS_* return codes.
7932 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7933 * wrong with the guest state.
7934 *
7935 * @param pVM Pointer to the VM.
7936 * @param pVCpu Pointer to the VMCPU.
7937 * @param pCtx Pointer to the guest-CPU state.
7938 */
7939static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7940{
7941#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7942#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
7943 uError = (err); \
7944 break; \
7945 } else do {} while (0)
7946/* Duplicate of IEM_IS_CANONICAL(). */
7947#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
7948
7949 int rc;
7950 uint64_t u64Val;
7951 uint32_t u32Val;
7952 uint32_t uError = VMX_IGS_ERROR;
7953 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
7954
7955 do
7956 {
7957 /*
7958 * CR0.
7959 */
7960 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
7961 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
7962 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
7963 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
7964 if (fUnrestrictedGuest)
7965 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
7966
7967 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7968 AssertRCBreak(rc);
7969 HMVMX_CHECK_BREAK((u32Val & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
7970 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR0), VMX_IGS_CR0_FIXED0);
7971 if ( !fUnrestrictedGuest
7972 && (u32Val & X86_CR0_PG)
7973 && !(u32Val & X86_CR0_PE))
7974 {
7975 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
7976 }
7977
7978 /*
7979 * CR4.
7980 */
7981 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
7982 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
7983 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7984 AssertRCBreak(rc);
7985 HMVMX_CHECK_BREAK((u32Val & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
7986 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR4), VMX_IGS_CR4_FIXED0);
7987
7988 /*
7989 * IA32_DEBUGCTL MSR.
7990 */
7991 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
7992 AssertRCBreak(rc);
7993 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7994 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
7995 {
7996 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
7997 }
7998 uint64_t u64DebugCtlMsr = u64Val;
7999
8000#ifdef VBOX_STRICT
8001 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
8002 AssertRCBreak(rc);
8003 Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
8004#endif
8005 bool const fLongModeGuest = !!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
8006
8007 /*
8008 * RIP and RFLAGS.
8009 */
8010 uint32_t u32Eflags;
8011#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8012 if (HMVMX_IS_64BIT_HOST_MODE())
8013 {
8014 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8015 AssertRCBreak(rc);
8016 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8017 if ( !fLongModeGuest
8018 || !pCtx->cs.Attr.n.u1Long)
8019 {
8020 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8021 }
8022 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8023 * must be identical if the "IA32e mode guest" VM-entry control is 1
8024 * and CS.L is 1. No check applies if the CPU supports 64
8025 * linear-address bits. */
8026
8027 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8028 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8029 AssertRCBreak(rc);
8030 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8031 VMX_IGS_RFLAGS_RESERVED);
8032 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8033 u32Eflags = u64Val;
8034 }
8035 else
8036#endif
8037 {
8038 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8039 AssertRCBreak(rc);
8040 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8041 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8042 }
8043
8044 if ( fLongModeGuest
8045 || !(pCtx->cr0 & X86_CR0_PE))
8046 {
8047 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8048 }
8049
8050 uint32_t u32EntryInfo;
8051 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8052 AssertRCBreak(rc);
8053 if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8054 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8055 {
8056 HMVMX_CHECK_BREAK(u32Val & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8057 }
8058
8059 /*
8060 * 64-bit checks.
8061 */
8062#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8063 if (HMVMX_IS_64BIT_HOST_MODE())
8064 {
8065 if ( fLongModeGuest
8066 && !fUnrestrictedGuest)
8067 {
8068 HMVMX_CHECK_BREAK(CPUMIsGuestPagingEnabledEx(pCtx), VMX_IGS_CR0_PG_LONGMODE);
8069 HMVMX_CHECK_BREAK((pCtx->cr4 & X86_CR4_PAE), VMX_IGS_CR4_PAE_LONGMODE);
8070 }
8071
8072 if ( !fLongModeGuest
8073 && (pCtx->cr4 & X86_CR4_PCIDE))
8074 {
8075 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8076 }
8077
8078 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8079 * 51:32 beyond the processor's physical-address width are 0. */
8080
8081 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8082 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8083 {
8084 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8085 }
8086
8087 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8088 AssertRCBreak(rc);
8089 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8090
8091 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8092 AssertRCBreak(rc);
8093 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8094 }
8095#endif
8096
8097 /*
8098 * PERF_GLOBAL MSR.
8099 */
8100 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8101 {
8102 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8103 AssertRCBreak(rc);
8104 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8105 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8106 }
8107
8108 /*
8109 * PAT MSR.
8110 */
8111 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8112 {
8113 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8114 AssertRCBreak(rc);
8115 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8116 for (unsigned i = 0; i < 8; i++)
8117 {
8118 uint8_t u8Val = (u64Val & 0x7);
8119 if ( u8Val != 0 /* UC */
8120 || u8Val != 1 /* WC */
8121 || u8Val != 4 /* WT */
8122 || u8Val != 5 /* WP */
8123 || u8Val != 6 /* WB */
8124 || u8Val != 7 /* UC- */)
8125 {
8126 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8127 }
8128 u64Val >>= 3;
8129 }
8130 }
8131
8132 /*
8133 * EFER MSR.
8134 */
8135 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8136 {
8137 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8138 AssertRCBreak(rc);
8139 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8140 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8141 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8142 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8143 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8144 || (u64Val & MSR_K6_EFER_LMA) == (pCtx->cr0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8145 }
8146
8147 /*
8148 * Segment registers.
8149 */
8150 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8151 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8152 if (!(u32Eflags & X86_EFL_VM))
8153 {
8154 /* CS */
8155 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8156 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8157 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8158 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8159 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8160 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8161 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8162 /* CS cannot be loaded with NULL in protected mode. */
8163 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8164 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8165 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8166 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8167 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8168 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8169 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8170 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8171 else
8172 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8173
8174 /* SS */
8175 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8176 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8177 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8178 if ( !(pCtx->cr0 & X86_CR0_PE)
8179 || pCtx->cs.Attr.n.u4Type == 3)
8180 {
8181 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8182 }
8183 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8184 {
8185 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8186 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8187 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8188 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8189 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8190 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8191 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8192 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8193 }
8194
8195 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8196 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8197 {
8198 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8199 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8200 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8201 || pCtx->ds.Attr.n.u4Type > 11
8202 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8203 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8204 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8205 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8206 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8207 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8208 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8209 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8210 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8211 }
8212 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8213 {
8214 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8215 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8216 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8217 || pCtx->es.Attr.n.u4Type > 11
8218 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8219 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8220 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8221 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8222 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8223 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8224 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8225 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8226 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8227 }
8228 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8229 {
8230 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8231 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8232 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8233 || pCtx->fs.Attr.n.u4Type > 11
8234 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8235 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8236 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8237 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8238 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8239 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8240 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8241 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8242 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8243 }
8244 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8245 {
8246 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8247 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8248 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8249 || pCtx->gs.Attr.n.u4Type > 11
8250 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8251 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8252 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8253 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8254 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8255 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8256 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8257 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8258 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8259 }
8260 /* 64-bit capable CPUs. */
8261#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8262 if (HMVMX_IS_64BIT_HOST_MODE())
8263 {
8264 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8265 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8266 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8267 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8268 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8269 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8270 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8271 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8272 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8273 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8274 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8275 }
8276#endif
8277 }
8278 else
8279 {
8280 /* V86 mode checks. */
8281 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8282 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8283 {
8284 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8285 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8286 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8287 }
8288 else
8289 {
8290 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8291 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8292 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8293 }
8294
8295 /* CS */
8296 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8297 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8298 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8299 /* SS */
8300 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8301 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8302 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8303 /* DS */
8304 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8305 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8306 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8307 /* ES */
8308 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8309 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8310 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8311 /* FS */
8312 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8313 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8314 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8315 /* GS */
8316 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8317 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8318 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8319 /* 64-bit capable CPUs. */
8320#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8321 if (HMVMX_IS_64BIT_HOST_MODE())
8322 {
8323 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8324 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8325 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8326 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8327 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8328 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8329 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8330 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8331 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8332 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8333 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8334 }
8335#endif
8336 }
8337
8338 /*
8339 * TR.
8340 */
8341 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8342 /* 64-bit capable CPUs. */
8343#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8344 if (HMVMX_IS_64BIT_HOST_MODE())
8345 {
8346 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8347 }
8348#endif
8349 if (fLongModeGuest)
8350 {
8351 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8352 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8353 }
8354 else
8355 {
8356 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8357 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8358 VMX_IGS_TR_ATTR_TYPE_INVALID);
8359 }
8360 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8361 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8362 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8363 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8364 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8365 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8366 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8367 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8368
8369 /*
8370 * GDTR and IDTR.
8371 */
8372#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8373 if (HMVMX_IS_64BIT_HOST_MODE())
8374 {
8375 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8376 AssertRCBreak(rc);
8377 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8378
8379 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8380 AssertRCBreak(rc);
8381 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8382 }
8383#endif
8384
8385 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8386 AssertRCBreak(rc);
8387 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8388
8389 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8390 AssertRCBreak(rc);
8391 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8392
8393 /*
8394 * Guest Non-Register State.
8395 */
8396 /* Activity State. */
8397 uint32_t u32ActivityState;
8398 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8399 AssertRCBreak(rc);
8400 HMVMX_CHECK_BREAK( !u32ActivityState
8401 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8402 VMX_IGS_ACTIVITY_STATE_INVALID);
8403 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8404 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8405 uint32_t u32IntrState;
8406 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8407 AssertRCBreak(rc);
8408 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8409 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8410 {
8411 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8412 }
8413
8414 /** @todo Activity state and injecting interrupts. Left as a todo since we
8415 * currently don't use activity states but ACTIVE. */
8416
8417 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8418 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8419
8420 /* Guest interruptibility-state. */
8421 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8422 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8423 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8424 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8425 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8426 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8427 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8428 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8429 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8430 if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8431 {
8432 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8433 {
8434 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8435 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8436 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8437 }
8438 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8439 {
8440 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8441 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8442 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8443 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8444 }
8445 }
8446 /** @todo Assumes the processor is not in SMM. */
8447 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8448 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8449 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8450 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8451 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8452 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8453 && VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8454 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8455 {
8456 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8457 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8458 }
8459
8460 /* Pending debug exceptions. */
8461 if (HMVMX_IS_64BIT_HOST_MODE())
8462 {
8463 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8464 AssertRCBreak(rc);
8465 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8466 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8467 u32Val = u64Val; /* For pending debug exceptions checks below. */
8468 }
8469 else
8470 {
8471 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8472 AssertRCBreak(rc);
8473 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8474 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8475 }
8476
8477 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8478 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8479 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8480 {
8481 if ( (u32Eflags & X86_EFL_TF)
8482 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8483 {
8484 /* Bit 14 is PendingDebug.BS. */
8485 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8486 }
8487 if ( !(u32Eflags & X86_EFL_TF)
8488 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8489 {
8490 /* Bit 14 is PendingDebug.BS. */
8491 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8492 }
8493 }
8494
8495 /* VMCS link pointer. */
8496 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8497 AssertRCBreak(rc);
8498 if (u64Val != UINT64_C(0xffffffffffffffff))
8499 {
8500 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8501 /** @todo Bits beyond the processor's physical-address width MBZ. */
8502 /** @todo 32-bit located in memory referenced by value of this field (as a
8503 * physical address) must contain the processor's VMCS revision ID. */
8504 /** @todo SMM checks. */
8505 }
8506
8507 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8508
8509 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8510 if (uError == VMX_IGS_ERROR)
8511 uError = VMX_IGS_REASON_NOT_FOUND;
8512 } while (0);
8513
8514 pVCpu->hm.s.u32HMError = uError;
8515 return uError;
8516
8517#undef HMVMX_ERROR_BREAK
8518#undef HMVMX_CHECK_BREAK
8519#undef HMVMX_IS_CANONICAL
8520}
8521
8522/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8523/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8524/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8525
8526/** @name VM-exit handlers.
8527 * @{
8528 */
8529
8530/**
8531 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8532 */
8533HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8534{
8535 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8536 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8537 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8538#if HC_ARCH_BITS == 64
8539 Assert(ASMIntAreEnabled());
8540 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8541 return VINF_SUCCESS;
8542#endif
8543 return VINF_EM_RAW_INTERRUPT;
8544}
8545
8546
8547/**
8548 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8549 */
8550HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8551{
8552 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8553 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8554
8555 int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8556 AssertRCReturn(rc, rc);
8557
8558 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8559 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8560 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8561 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8562
8563 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8564 {
8565 /*
8566 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8567 * anything we inject is not going to cause a VM-exit directly for the event being injected.
8568 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8569 *
8570 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8571 */
8572 VMXDispatchHostNmi();
8573 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmi);
8574 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8575 return VINF_SUCCESS;
8576 }
8577
8578 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8579 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8580 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8581 {
8582 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8583 return VINF_SUCCESS;
8584 }
8585 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8586 {
8587 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8588 return rc;
8589 }
8590
8591 uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8592 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8593 switch (uIntrType)
8594 {
8595 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8596 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8597 /* no break */
8598 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8599 {
8600 switch (uVector)
8601 {
8602 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8603 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8604 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8605 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8606 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8607 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8608#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8609 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8610 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8611 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8612 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8613 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8614 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8615 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8616 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8617 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8618 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8619#endif
8620 default:
8621 {
8622 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8623 AssertRCReturn(rc, rc);
8624
8625 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8626 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8627 {
8628 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8629 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8630 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8631
8632 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8633 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8634 AssertRCReturn(rc, rc);
8635 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8636 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8637 0 /* GCPtrFaultAddress */);
8638 AssertRCReturn(rc, rc);
8639 }
8640 else
8641 {
8642 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8643 pVCpu->hm.s.u32HMError = uVector;
8644 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8645 }
8646 break;
8647 }
8648 }
8649 break;
8650 }
8651
8652 default:
8653 {
8654 pVCpu->hm.s.u32HMError = uExitIntrInfo;
8655 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8656 AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8657 break;
8658 }
8659 }
8660 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8661 return rc;
8662}
8663
8664
8665/**
8666 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8667 */
8668HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8669{
8670 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8671
8672 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8673 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8674 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8675 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8676 AssertRCReturn(rc, rc);
8677
8678 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8679 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8680 return VINF_SUCCESS;
8681}
8682
8683
8684/**
8685 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8686 */
8687HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8688{
8689 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8690 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8691 pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8692 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8693}
8694
8695
8696/**
8697 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8698 */
8699HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8700{
8701 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8702 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8703 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8704}
8705
8706
8707/**
8708 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8709 */
8710HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8711{
8712 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8713 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8714 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8715}
8716
8717
8718/**
8719 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8720 */
8721HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8722{
8723 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8724 PVM pVM = pVCpu->CTX_SUFF(pVM);
8725 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8726 if (RT_LIKELY(rc == VINF_SUCCESS))
8727 {
8728 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8729 Assert(pVmxTransient->cbInstr == 2);
8730 }
8731 else
8732 {
8733 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8734 rc = VERR_EM_INTERPRETER;
8735 }
8736 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8737 return rc;
8738}
8739
8740
8741/**
8742 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8743 */
8744HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8745{
8746 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8747 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8748 AssertRCReturn(rc, rc);
8749
8750 if (pMixedCtx->cr4 & X86_CR4_SMXE)
8751 return VINF_EM_RAW_EMULATE_INSTR;
8752
8753 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8754 pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8755 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8756}
8757
8758
8759/**
8760 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8761 */
8762HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8763{
8764 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8765 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8766 AssertRCReturn(rc, rc);
8767
8768 PVM pVM = pVCpu->CTX_SUFF(pVM);
8769 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8770 if (RT_LIKELY(rc == VINF_SUCCESS))
8771 {
8772 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8773 Assert(pVmxTransient->cbInstr == 2);
8774 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8775 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8776 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8777 }
8778 else
8779 {
8780 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8781 rc = VERR_EM_INTERPRETER;
8782 }
8783 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8784 return rc;
8785}
8786
8787
8788/**
8789 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8790 */
8791HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8792{
8793 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8794 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8795 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8796 AssertRCReturn(rc, rc);
8797
8798 PVM pVM = pVCpu->CTX_SUFF(pVM);
8799 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8800 if (RT_LIKELY(rc == VINF_SUCCESS))
8801 {
8802 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8803 Assert(pVmxTransient->cbInstr == 3);
8804 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8805 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8806 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8807 }
8808 else
8809 {
8810 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8811 rc = VERR_EM_INTERPRETER;
8812 }
8813 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8814 return rc;
8815}
8816
8817
8818/**
8819 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8820 */
8821HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8822{
8823 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8824 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8825 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8826 AssertRCReturn(rc, rc);
8827
8828 PVM pVM = pVCpu->CTX_SUFF(pVM);
8829 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8830 if (RT_LIKELY(rc == VINF_SUCCESS))
8831 {
8832 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8833 Assert(pVmxTransient->cbInstr == 2);
8834 }
8835 else
8836 {
8837 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8838 rc = VERR_EM_INTERPRETER;
8839 }
8840 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8841 return rc;
8842}
8843
8844
8845/**
8846 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8847 */
8848HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8849{
8850 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8851 PVM pVM = pVCpu->CTX_SUFF(pVM);
8852 Assert(!pVM->hm.s.fNestedPaging);
8853
8854 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8855 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8856 AssertRCReturn(rc, rc);
8857
8858 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8859 rc = VBOXSTRICTRC_VAL(rc2);
8860 if (RT_LIKELY(rc == VINF_SUCCESS))
8861 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8862 else
8863 {
8864 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8865 pVmxTransient->uExitQualification, rc));
8866 }
8867 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8868 return rc;
8869}
8870
8871
8872/**
8873 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8874 */
8875HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8876{
8877 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8878 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8879 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8880 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8881 AssertRCReturn(rc, rc);
8882
8883 PVM pVM = pVCpu->CTX_SUFF(pVM);
8884 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8885 if (RT_LIKELY(rc == VINF_SUCCESS))
8886 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8887 else
8888 {
8889 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8890 rc = VERR_EM_INTERPRETER;
8891 }
8892 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8893 return rc;
8894}
8895
8896
8897/**
8898 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8899 */
8900HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8901{
8902 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8903 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8904 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8905 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8906 AssertRCReturn(rc, rc);
8907
8908 PVM pVM = pVCpu->CTX_SUFF(pVM);
8909 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8910 rc = VBOXSTRICTRC_VAL(rc2);
8911 if (RT_LIKELY( rc == VINF_SUCCESS
8912 || rc == VINF_EM_HALT))
8913 {
8914 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8915 AssertRCReturn(rc3, rc3);
8916
8917 if ( rc == VINF_EM_HALT
8918 && EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
8919 {
8920 rc = VINF_SUCCESS;
8921 }
8922 }
8923 else
8924 {
8925 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
8926 rc = VERR_EM_INTERPRETER;
8927 }
8928 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
8929 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
8930 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
8931 return rc;
8932}
8933
8934
8935/**
8936 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
8937 */
8938HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8939{
8940 /*
8941 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
8942 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
8943 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
8944 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
8945 */
8946 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8947 pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
8948 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8949}
8950
8951
8952/**
8953 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
8954 */
8955HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8956{
8957 /*
8958 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
8959 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
8960 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
8961 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
8962 */
8963 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8964 pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
8965 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8966}
8967
8968
8969/**
8970 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
8971 */
8972HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8973{
8974 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
8975 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8976 pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
8977 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8978}
8979
8980
8981/**
8982 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
8983 */
8984HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8985{
8986 /*
8987 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
8988 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
8989 * See Intel spec. 25.3 "Other Causes of VM-exits".
8990 */
8991 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8992 pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
8993 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8994}
8995
8996
8997/**
8998 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
8999 * VM-exit.
9000 */
9001HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9002{
9003 /*
9004 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
9005 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
9006 *
9007 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
9008 * See Intel spec. "23.8 Restrictions on VMX operation".
9009 */
9010 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9011 return VINF_SUCCESS;
9012}
9013
9014
9015/**
9016 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9017 * VM-exit.
9018 */
9019HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9020{
9021 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9022 return VINF_EM_RESET;
9023}
9024
9025
9026/**
9027 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9028 */
9029HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9030{
9031 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9032 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9033 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9034 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9035 AssertRCReturn(rc, rc);
9036
9037 pMixedCtx->rip++;
9038 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9039 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9040 rc = VINF_SUCCESS;
9041 else
9042 rc = VINF_EM_HALT;
9043
9044 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9045 return rc;
9046}
9047
9048
9049/**
9050 * VM-exit handler for instructions that result in a #UD exception delivered to
9051 * the guest.
9052 */
9053HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9054{
9055 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9056 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9057 return VINF_SUCCESS;
9058}
9059
9060
9061/**
9062 * VM-exit handler for expiry of the VMX preemption timer.
9063 */
9064HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9065{
9066 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9067
9068 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9069 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9070
9071 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9072 PVM pVM = pVCpu->CTX_SUFF(pVM);
9073 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9074 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9075 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9076}
9077
9078
9079/**
9080 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9081 */
9082HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9083{
9084 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9085
9086 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9087 /** @todo check if XSETBV is supported by the recompiler. */
9088 return VERR_EM_INTERPRETER;
9089}
9090
9091
9092/**
9093 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9094 */
9095HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9096{
9097 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9098
9099 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9100 /** @todo implement EMInterpretInvpcid() */
9101 return VERR_EM_INTERPRETER;
9102}
9103
9104
9105/**
9106 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9107 * Error VM-exit.
9108 */
9109HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9110{
9111 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9112 AssertRCReturn(rc, rc);
9113
9114 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9115 NOREF(uInvalidReason);
9116
9117#ifdef VBOX_STRICT
9118 uint32_t uIntrState;
9119 HMVMXHCUINTREG uHCReg;
9120 uint64_t u64Val;
9121 uint32_t u32Val;
9122
9123 rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
9124 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9125 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9126 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9127 AssertRCReturn(rc, rc);
9128
9129 Log4(("uInvalidReason %u\n", uInvalidReason));
9130 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
9131 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9132 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9133 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9134
9135 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9136 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9137 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9138 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9139 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9140 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9141 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9142 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9143 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9144 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9145 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9146 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9147#endif
9148
9149 PVM pVM = pVCpu->CTX_SUFF(pVM);
9150 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9151
9152 return VERR_VMX_INVALID_GUEST_STATE;
9153}
9154
9155
9156/**
9157 * VM-exit handler for VM-entry failure due to an MSR-load
9158 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9159 */
9160HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9161{
9162 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9163 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9164}
9165
9166
9167/**
9168 * VM-exit handler for VM-entry failure due to a machine-check event
9169 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9170 */
9171HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9172{
9173 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9174 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9175}
9176
9177
9178/**
9179 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9180 * theory.
9181 */
9182HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9183{
9184 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9185 return VERR_VMX_UNDEFINED_EXIT_CODE;
9186}
9187
9188
9189/**
9190 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9191 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9192 * Conditional VM-exit.
9193 */
9194HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9195{
9196 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9197
9198 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9199 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9200 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9201 return VERR_EM_INTERPRETER;
9202 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9203 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9204}
9205
9206
9207/**
9208 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9209 */
9210HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9211{
9212 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9213
9214 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9215 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9216 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9217 return VERR_EM_INTERPRETER;
9218 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9219 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9220}
9221
9222
9223/**
9224 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9225 */
9226HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9227{
9228 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9229
9230 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9231 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9232 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9233 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9234 AssertRCReturn(rc, rc);
9235 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9236
9237 PVM pVM = pVCpu->CTX_SUFF(pVM);
9238 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9239 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9240 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9241 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9242
9243 if (RT_LIKELY(rc == VINF_SUCCESS))
9244 {
9245 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9246 Assert(pVmxTransient->cbInstr == 2);
9247 }
9248 return rc;
9249}
9250
9251
9252/**
9253 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9254 */
9255HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9256{
9257 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9258 PVM pVM = pVCpu->CTX_SUFF(pVM);
9259 int rc = VINF_SUCCESS;
9260
9261 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9262 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9263 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9264 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9265 AssertRCReturn(rc, rc);
9266 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9267
9268 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9269 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9270 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9271
9272 if (RT_LIKELY(rc == VINF_SUCCESS))
9273 {
9274 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9275
9276 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9277 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9278 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9279 {
9280 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9281 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9282 EMInterpretWrmsr() changes it. */
9283 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9284 }
9285 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9286 {
9287 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9288 AssertRCReturn(rc, rc);
9289 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
9290 }
9291 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9292 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9293
9294 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9295 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9296 {
9297 switch (pMixedCtx->ecx)
9298 {
9299 case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
9300 case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
9301 case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
9302 case MSR_K8_FS_BASE: /* no break */
9303 case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS; break;
9304 case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
9305 }
9306 }
9307#ifdef VBOX_STRICT
9308 else
9309 {
9310 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9311 switch (pMixedCtx->ecx)
9312 {
9313 case MSR_IA32_SYSENTER_CS:
9314 case MSR_IA32_SYSENTER_EIP:
9315 case MSR_IA32_SYSENTER_ESP:
9316 case MSR_K8_FS_BASE:
9317 case MSR_K8_GS_BASE:
9318 {
9319 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9320 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9321 }
9322
9323 case MSR_K8_LSTAR:
9324 case MSR_K6_STAR:
9325 case MSR_K8_SF_MASK:
9326 case MSR_K8_TSC_AUX:
9327 case MSR_K8_KERNEL_GS_BASE:
9328 {
9329 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9330 pMixedCtx->ecx));
9331 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9332 }
9333 }
9334 }
9335#endif /* VBOX_STRICT */
9336 }
9337 return rc;
9338}
9339
9340
9341/**
9342 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9343 */
9344HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9345{
9346 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9347
9348 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9349 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9350 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9351 return VERR_EM_INTERPRETER;
9352 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9353 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9354}
9355
9356
9357/**
9358 * VM-exit handler for when the TPR value is lowered below the specified
9359 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9360 */
9361HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9362{
9363 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9364 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9365
9366 /*
9367 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9368 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9369 * resume guest execution.
9370 */
9371 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9372 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9373 return VINF_SUCCESS;
9374}
9375
9376
9377/**
9378 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9379 * VM-exit.
9380 *
9381 * @retval VINF_SUCCESS when guest execution can continue.
9382 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9383 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9384 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9385 * recompiler.
9386 */
9387HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9388{
9389 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9390 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9391 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9392 AssertRCReturn(rc, rc);
9393
9394 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9395 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9396 PVM pVM = pVCpu->CTX_SUFF(pVM);
9397 switch (uAccessType)
9398 {
9399 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9400 {
9401#if 0
9402 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9403 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9404#else
9405 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9406 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9407 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9408#endif
9409 AssertRCReturn(rc, rc);
9410
9411 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9412 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9413 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9414 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9415
9416 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9417 {
9418 case 0: /* CR0 */
9419 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9420 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9421 break;
9422 case 2: /* C2 **/
9423 /* Nothing to do here, CR2 it's not part of the VMCS. */
9424 break;
9425 case 3: /* CR3 */
9426 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9427 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9428 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR3;
9429 break;
9430 case 4: /* CR4 */
9431 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9432 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR4;
9433 break;
9434 case 8: /* CR8 */
9435 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9436 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9437 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9438 break;
9439 default:
9440 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9441 break;
9442 }
9443
9444 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9445 break;
9446 }
9447
9448 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9449 {
9450 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9451 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9452 AssertRCReturn(rc, rc);
9453 Assert( !pVM->hm.s.fNestedPaging
9454 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9455 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9456
9457 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9458 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9459 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9460
9461 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9462 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9463 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9464 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9465 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9466 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9467 break;
9468 }
9469
9470 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9471 {
9472 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9473 AssertRCReturn(rc, rc);
9474 rc = EMInterpretCLTS(pVM, pVCpu);
9475 AssertRCReturn(rc, rc);
9476 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9477 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9478 Log4(("CRX CLTS write rc=%d\n", rc));
9479 break;
9480 }
9481
9482 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9483 {
9484 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9485 AssertRCReturn(rc, rc);
9486 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9487 if (RT_LIKELY(rc == VINF_SUCCESS))
9488 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9489 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9490 Log4(("CRX LMSW write rc=%d\n", rc));
9491 break;
9492 }
9493
9494 default:
9495 {
9496 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9497 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9498 }
9499 }
9500
9501 /* Validate possible error codes. */
9502 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9503 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9504 if (RT_SUCCESS(rc))
9505 {
9506 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9507 AssertRCReturn(rc2, rc2);
9508 }
9509
9510 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9511 return rc;
9512}
9513
9514
9515/**
9516 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9517 * VM-exit.
9518 */
9519HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9520{
9521 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9522 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9523
9524 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9525 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9526 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9527 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9528 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9529 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9530 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9531 AssertRCReturn(rc2, rc2);
9532
9533 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9534 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9535 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9536 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9537 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9538 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9539 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9540
9541 /* I/O operation lookup arrays. */
9542 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9543 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9544
9545 VBOXSTRICTRC rcStrict;
9546 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9547 const uint32_t cbInstr = pVmxTransient->cbInstr;
9548 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9549 PVM pVM = pVCpu->CTX_SUFF(pVM);
9550 if (fIOString)
9551 {
9552 /*
9553 * INS/OUTS - I/O String instruction.
9554 *
9555 * Use instruction-information if available, otherwise fall back on
9556 * interpreting the instruction.
9557 */
9558 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9559#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9560 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9561 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9562 {
9563 rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9564 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9565 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9566 AssertRCReturn(rc2, rc2);
9567 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9568 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9569 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9570 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9571 if (fIOWrite)
9572 {
9573 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9574 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9575 //if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9576 // hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9577 // pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9578 }
9579 else
9580 {
9581 AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9582 ("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9583 VERR_HMVMX_IPE_4);
9584 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9585 //if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9586 // hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9587 }
9588 }
9589 else
9590 {
9591 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9592 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9593 AssertRCReturn(rc2, rc2);
9594 rcStrict = IEMExecOne(pVCpu);
9595 }
9596 /** @todo IEM needs to be setting these flags somehow. */
9597 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9598 fUpdateRipAlready = true;
9599#else
9600 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9601 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9602 if (RT_SUCCESS(rcStrict))
9603 {
9604 if (fIOWrite)
9605 {
9606 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9607 (DISCPUMODE)pDis->uAddrMode, cbValue);
9608 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9609 }
9610 else
9611 {
9612 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9613 (DISCPUMODE)pDis->uAddrMode, cbValue);
9614 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9615 }
9616 }
9617 else
9618 {
9619 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9620 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9621 }
9622#endif
9623 }
9624 else
9625 {
9626 /*
9627 * IN/OUT - I/O instruction.
9628 */
9629 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9630 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9631 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9632 if (fIOWrite)
9633 {
9634 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9635 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9636 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9637 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9638 }
9639 else
9640 {
9641 uint32_t u32Result = 0;
9642 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9643 if (IOM_SUCCESS(rcStrict))
9644 {
9645 /* Save result of I/O IN instr. in AL/AX/EAX. */
9646 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
9647 }
9648 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9649 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9650 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9651 }
9652 }
9653
9654 if (IOM_SUCCESS(rcStrict))
9655 {
9656 if (!fUpdateRipAlready)
9657 {
9658 pMixedCtx->rip += cbInstr;
9659 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9660 }
9661
9662 /*
9663 * If any I/O breakpoints are armed, we need to check if one triggered
9664 * and take appropriate action.
9665 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9666 */
9667 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9668 AssertRCReturn(rc2, rc2);
9669
9670 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9671 * execution engines about whether hyper BPs and such are pending. */
9672 uint32_t const uDr7 = pMixedCtx->dr[7];
9673 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9674 && X86_DR7_ANY_RW_IO(uDr7)
9675 && (pMixedCtx->cr4 & X86_CR4_DE))
9676 || DBGFBpIsHwIoArmed(pVM)))
9677 {
9678 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9679
9680 /* We're playing with the host CPU state here, make sure we don't preempt. */
9681 HM_DISABLE_PREEMPT_IF_NEEDED();
9682 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
9683
9684 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9685 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9686 {
9687 /* Raise #DB. */
9688 if (fIsGuestDbgActive)
9689 ASMSetDR6(pMixedCtx->dr[6]);
9690 if (pMixedCtx->dr[7] != uDr7)
9691 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9692
9693 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9694 }
9695 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9696 else if ( rcStrict2 != VINF_SUCCESS
9697 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
9698 rcStrict = rcStrict2;
9699
9700 HM_RESTORE_PREEMPT_IF_NEEDED();
9701 }
9702 }
9703
9704#ifdef DEBUG
9705 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9706 Assert(!fIOWrite);
9707 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9708 Assert(fIOWrite);
9709 else
9710 {
9711 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9712 * statuses, that the VMM device and some others may return. See
9713 * IOM_SUCCESS() for guidance. */
9714 AssertMsg( RT_FAILURE(rcStrict)
9715 || rcStrict == VINF_SUCCESS
9716 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
9717 || rcStrict == VINF_EM_DBG_BREAKPOINT
9718 || rcStrict == VINF_EM_RAW_GUEST_TRAP
9719 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9720 }
9721#endif
9722
9723 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9724 return VBOXSTRICTRC_TODO(rcStrict);
9725}
9726
9727
9728/**
9729 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9730 * VM-exit.
9731 */
9732HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9733{
9734 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9735
9736 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
9737 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9738 AssertRCReturn(rc, rc);
9739 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9740 {
9741 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9742 AssertRCReturn(rc, rc);
9743 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9744 {
9745 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9746
9747 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9748 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9749 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9750 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9751 {
9752 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9753 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9754
9755 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9756 Assert(!pVCpu->hm.s.Event.fPending);
9757 pVCpu->hm.s.Event.fPending = true;
9758 pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9759 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9760 AssertRCReturn(rc, rc);
9761 if (fErrorCodeValid)
9762 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9763 else
9764 pVCpu->hm.s.Event.u32ErrCode = 0;
9765 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9766 && uVector == X86_XCPT_PF)
9767 {
9768 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9769 }
9770
9771 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9772 }
9773 }
9774 }
9775
9776 /** @todo Emulate task switch someday, currently just going back to ring-3 for
9777 * emulation. */
9778 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9779 return VERR_EM_INTERPRETER;
9780}
9781
9782
9783/**
9784 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9785 */
9786HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9787{
9788 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9789 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9790 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9791 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9792 AssertRCReturn(rc, rc);
9793 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9794 return VINF_EM_DBG_STEPPED;
9795}
9796
9797
9798/**
9799 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9800 */
9801HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9802{
9803 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9804
9805 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9806 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9807 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9808 return VINF_SUCCESS;
9809 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9810 return rc;
9811
9812#if 0
9813 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9814 * just sync the whole thing. */
9815 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9816#else
9817 /* Aggressive state sync. for now. */
9818 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9819 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9820 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9821#endif
9822 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9823 AssertRCReturn(rc, rc);
9824
9825 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9826 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9827 switch (uAccessType)
9828 {
9829 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9830 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9831 {
9832 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9833 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9834 {
9835 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9836 }
9837
9838 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9839 GCPhys &= PAGE_BASE_GC_MASK;
9840 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9841 PVM pVM = pVCpu->CTX_SUFF(pVM);
9842 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9843 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9844
9845 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9846 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9847 CPUMCTX2CORE(pMixedCtx), GCPhys);
9848 rc = VBOXSTRICTRC_VAL(rc2);
9849 Log4(("ApicAccess rc=%d\n", rc));
9850 if ( rc == VINF_SUCCESS
9851 || rc == VERR_PAGE_TABLE_NOT_PRESENT
9852 || rc == VERR_PAGE_NOT_PRESENT)
9853 {
9854 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
9855 | HM_CHANGED_VMX_GUEST_APIC_STATE;
9856 rc = VINF_SUCCESS;
9857 }
9858 break;
9859 }
9860
9861 default:
9862 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9863 rc = VINF_EM_RAW_EMULATE_INSTR;
9864 break;
9865 }
9866
9867 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9868 return rc;
9869}
9870
9871
9872/**
9873 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9874 * VM-exit.
9875 */
9876HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9877{
9878 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9879
9880 /* We should -not- get this VM-exit if the guest is debugging. */
9881 if (CPUMIsGuestDebugStateActive(pVCpu))
9882 {
9883 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9884 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9885 }
9886
9887 int rc = VERR_INTERNAL_ERROR_5;
9888 if ( !DBGFIsStepping(pVCpu)
9889 && !pVCpu->hm.s.fSingleInstruction
9890 && !CPUMIsHyperDebugStateActive(pVCpu))
9891 {
9892 /* Don't intercept MOV DRx and #DB any more. */
9893 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
9894 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9895 AssertRCReturn(rc, rc);
9896
9897 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9898 {
9899#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9900 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
9901 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9902 AssertRCReturn(rc, rc);
9903#endif
9904 }
9905
9906 /* We're playing with the host CPU state here, make sure we can't preempt. */
9907 HM_DISABLE_PREEMPT_IF_NEEDED();
9908
9909 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9910 PVM pVM = pVCpu->CTX_SUFF(pVM);
9911 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9912 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
9913
9914 HM_RESTORE_PREEMPT_IF_NEEDED();
9915
9916#ifdef VBOX_WITH_STATISTICS
9917 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9918 AssertRCReturn(rc, rc);
9919 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9920 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9921 else
9922 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9923#endif
9924 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
9925 return VINF_SUCCESS;
9926 }
9927
9928 /*
9929 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
9930 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
9931 */
9932 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9933 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9934 AssertRCReturn(rc, rc);
9935 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9936
9937 PVM pVM = pVCpu->CTX_SUFF(pVM);
9938 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9939 {
9940 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9941 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
9942 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
9943 if (RT_SUCCESS(rc))
9944 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9945 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9946 }
9947 else
9948 {
9949 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9950 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
9951 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
9952 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9953 }
9954
9955 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9956 if (RT_SUCCESS(rc))
9957 {
9958 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9959 AssertRCReturn(rc2, rc2);
9960 }
9961 return rc;
9962}
9963
9964
9965/**
9966 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
9967 * Conditional VM-exit.
9968 */
9969HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9970{
9971 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9972 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9973
9974 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9975 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9976 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9977 return VINF_SUCCESS;
9978 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9979 return rc;
9980
9981 RTGCPHYS GCPhys = 0;
9982 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9983
9984#if 0
9985 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9986#else
9987 /* Aggressive state sync. for now. */
9988 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9989 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9990 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9991#endif
9992 AssertRCReturn(rc, rc);
9993
9994 /*
9995 * If we succeed, resume guest execution.
9996 * If we fail in interpreting the instruction because we couldn't get the guest physical address
9997 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
9998 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
9999 * weird case. See @bugref{6043}.
10000 */
10001 PVM pVM = pVCpu->CTX_SUFF(pVM);
10002 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
10003 rc = VBOXSTRICTRC_VAL(rc2);
10004 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
10005 if ( rc == VINF_SUCCESS
10006 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10007 || rc == VERR_PAGE_NOT_PRESENT)
10008 {
10009 /* Successfully handled MMIO operation. */
10010 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10011 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10012 rc = VINF_SUCCESS;
10013 }
10014 return rc;
10015}
10016
10017
10018/**
10019 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10020 * VM-exit.
10021 */
10022HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10023{
10024 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10025 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10026
10027 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10028 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10029 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10030 return VINF_SUCCESS;
10031 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10032 return rc;
10033
10034 RTGCPHYS GCPhys = 0;
10035 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10036 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10037#if 0
10038 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10039#else
10040 /* Aggressive state sync. for now. */
10041 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10042 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10043 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10044#endif
10045 AssertRCReturn(rc, rc);
10046
10047 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10048 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10049
10050 RTGCUINT uErrorCode = 0;
10051 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10052 uErrorCode |= X86_TRAP_PF_ID;
10053 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10054 uErrorCode |= X86_TRAP_PF_RW;
10055 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10056 uErrorCode |= X86_TRAP_PF_P;
10057
10058 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10059
10060 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10061 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10062
10063 /* Handle the pagefault trap for the nested shadow table. */
10064 PVM pVM = pVCpu->CTX_SUFF(pVM);
10065 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10066 TRPMResetTrap(pVCpu);
10067
10068 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10069 if ( rc == VINF_SUCCESS
10070 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10071 || rc == VERR_PAGE_NOT_PRESENT)
10072 {
10073 /* Successfully synced our nested page tables. */
10074 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10075 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10076 return VINF_SUCCESS;
10077 }
10078
10079 Log4(("EPT return to ring-3 rc=%d\n"));
10080 return rc;
10081}
10082
10083/** @} */
10084
10085/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10086/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10087/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10088
10089/** @name VM-exit exception handlers.
10090 * @{
10091 */
10092
10093/**
10094 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10095 */
10096static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10097{
10098 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10099 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10100
10101 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10102 AssertRCReturn(rc, rc);
10103
10104 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10105 {
10106 /* Old-style FPU error reporting needs some extra work. */
10107 /** @todo don't fall back to the recompiler, but do it manually. */
10108 return VERR_EM_INTERPRETER;
10109 }
10110
10111 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10112 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10113 return rc;
10114}
10115
10116
10117/**
10118 * VM-exit exception handler for #BP (Breakpoint exception).
10119 */
10120static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10121{
10122 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10123 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10124
10125 /** @todo Try optimize this by not saving the entire guest state unless
10126 * really needed. */
10127 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10128 AssertRCReturn(rc, rc);
10129
10130 PVM pVM = pVCpu->CTX_SUFF(pVM);
10131 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10132 if (rc == VINF_EM_RAW_GUEST_TRAP)
10133 {
10134 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10135 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10136 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10137 AssertRCReturn(rc, rc);
10138
10139 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10140 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10141 }
10142
10143 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10144 return rc;
10145}
10146
10147
10148/**
10149 * VM-exit exception handler for #DB (Debug exception).
10150 */
10151static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10152{
10153 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10154 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10155 Log6(("XcptDB\n"));
10156
10157 /*
10158 * Get the DR6-like values from the exit qualification and pass it to DBGF
10159 * for processing.
10160 */
10161 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10162 AssertRCReturn(rc, rc);
10163
10164 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10165 uint64_t uDR6 = X86_DR6_INIT_VAL;
10166 uDR6 |= ( pVmxTransient->uExitQualification
10167 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10168
10169 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10170 if (rc == VINF_EM_RAW_GUEST_TRAP)
10171 {
10172 /*
10173 * The exception was for the guest. Update DR6, DR7.GD and
10174 * IA32_DEBUGCTL.LBR before forwarding it.
10175 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10176 */
10177 HM_DISABLE_PREEMPT_IF_NEEDED();
10178
10179 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10180 pMixedCtx->dr[6] |= uDR6;
10181 if (CPUMIsGuestDebugStateActive(pVCpu))
10182 ASMSetDR6(pMixedCtx->dr[6]);
10183
10184 HM_RESTORE_PREEMPT_IF_NEEDED();
10185
10186 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10187 AssertRCReturn(rc, rc);
10188
10189 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10190 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10191
10192 /* Paranoia. */
10193 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10194 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10195
10196 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10197 AssertRCReturn(rc, rc);
10198
10199 /*
10200 * Raise #DB in the guest.
10201 */
10202 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10203 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10204 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10205 AssertRCReturn(rc, rc);
10206 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10207 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10208 return VINF_SUCCESS;
10209 }
10210
10211 /*
10212 * Not a guest trap, must be a hypervisor related debug event then.
10213 * Update DR6 in case someone is interested in it.
10214 */
10215 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10216 AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
10217 CPUMSetHyperDR6(pVCpu, uDR6);
10218
10219 return rc;
10220}
10221
10222
10223/**
10224 * VM-exit exception handler for #NM (Device-not-available exception: floating
10225 * point exception).
10226 */
10227static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10228{
10229 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10230
10231#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10232 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
10233#endif
10234
10235 /* We require CR0 and EFER. EFER is always up-to-date. */
10236 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10237 AssertRCReturn(rc, rc);
10238
10239 /* We're playing with the host CPU state here, have to disable preemption. */
10240 HM_DISABLE_PREEMPT_IF_NEEDED();
10241
10242 /* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10243 PVM pVM = pVCpu->CTX_SUFF(pVM);
10244 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
10245 if (rc == VINF_SUCCESS)
10246 {
10247 Assert(CPUMIsGuestFPUStateActive(pVCpu));
10248 HM_RESTORE_PREEMPT_IF_NEEDED();
10249
10250 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
10251 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10252 return VINF_SUCCESS;
10253 }
10254
10255 HM_RESTORE_PREEMPT_IF_NEEDED();
10256
10257 /* Forward #NM to the guest. */
10258 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10259 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10260 AssertRCReturn(rc, rc);
10261 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10262 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10264 return rc;
10265}
10266
10267
10268/**
10269 * VM-exit exception handler for #GP (General-protection exception).
10270 *
10271 * @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
10272 */
10273static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10274{
10275 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10276 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10277
10278 int rc = VERR_INTERNAL_ERROR_5;
10279 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10280 {
10281#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10282 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10283 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10284 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10285 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10286 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10287 AssertRCReturn(rc, rc);
10288 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10289 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10290 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10291 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10292 return rc;
10293#else
10294 /* We don't intercept #GP. */
10295 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10296 return VERR_VMX_UNEXPECTED_EXCEPTION;
10297#endif
10298 }
10299
10300 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10301 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10302
10303 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10304 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10305 AssertRCReturn(rc, rc);
10306
10307 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10308 uint32_t cbOp = 0;
10309 PVM pVM = pVCpu->CTX_SUFF(pVM);
10310 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10311 if (RT_SUCCESS(rc))
10312 {
10313 rc = VINF_SUCCESS;
10314 Assert(cbOp == pDis->cbInstr);
10315 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10316 switch (pDis->pCurInstr->uOpcode)
10317 {
10318 case OP_CLI:
10319 {
10320 pMixedCtx->eflags.Bits.u1IF = 0;
10321 pMixedCtx->rip += pDis->cbInstr;
10322 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10323 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10324 break;
10325 }
10326
10327 case OP_STI:
10328 {
10329 pMixedCtx->eflags.Bits.u1IF = 1;
10330 pMixedCtx->rip += pDis->cbInstr;
10331 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10332 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10333 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10334 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10335 break;
10336 }
10337
10338 case OP_HLT:
10339 {
10340 rc = VINF_EM_HALT;
10341 pMixedCtx->rip += pDis->cbInstr;
10342 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
10343 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10344 break;
10345 }
10346
10347 case OP_POPF:
10348 {
10349 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10350 uint32_t cbParm = 0;
10351 uint32_t uMask = 0;
10352 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10353 {
10354 cbParm = 4;
10355 uMask = 0xffffffff;
10356 }
10357 else
10358 {
10359 cbParm = 2;
10360 uMask = 0xffff;
10361 }
10362
10363 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10364 RTGCPTR GCPtrStack = 0;
10365 X86EFLAGS Eflags;
10366 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10367 &GCPtrStack);
10368 if (RT_SUCCESS(rc))
10369 {
10370 Assert(sizeof(Eflags.u32) >= cbParm);
10371 Eflags.u32 = 0;
10372 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10373 }
10374 if (RT_FAILURE(rc))
10375 {
10376 rc = VERR_EM_INTERPRETER;
10377 break;
10378 }
10379 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10380 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10381 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10382 /* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10383 pMixedCtx->eflags.Bits.u1RF = 0;
10384 pMixedCtx->esp += cbParm;
10385 pMixedCtx->esp &= uMask;
10386 pMixedCtx->rip += pDis->cbInstr;
10387 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10388 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10389 break;
10390 }
10391
10392 case OP_PUSHF:
10393 {
10394 uint32_t cbParm = 0;
10395 uint32_t uMask = 0;
10396 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10397 {
10398 cbParm = 4;
10399 uMask = 0xffffffff;
10400 }
10401 else
10402 {
10403 cbParm = 2;
10404 uMask = 0xffff;
10405 }
10406
10407 /* Get the stack pointer & push the contents of eflags onto the stack. */
10408 RTGCPTR GCPtrStack = 0;
10409 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10410 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10411 if (RT_FAILURE(rc))
10412 {
10413 rc = VERR_EM_INTERPRETER;
10414 break;
10415 }
10416 X86EFLAGS Eflags = pMixedCtx->eflags;
10417 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10418 Eflags.Bits.u1RF = 0;
10419 Eflags.Bits.u1VM = 0;
10420
10421 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10422 if (RT_FAILURE(rc))
10423 {
10424 rc = VERR_EM_INTERPRETER;
10425 break;
10426 }
10427 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10428 pMixedCtx->esp -= cbParm;
10429 pMixedCtx->esp &= uMask;
10430 pMixedCtx->rip += pDis->cbInstr;
10431 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP;
10432 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10433 break;
10434 }
10435
10436 case OP_IRET:
10437 {
10438 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10439 * instruction reference. */
10440 RTGCPTR GCPtrStack = 0;
10441 uint32_t uMask = 0xffff;
10442 uint16_t aIretFrame[3];
10443 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10444 {
10445 rc = VERR_EM_INTERPRETER;
10446 break;
10447 }
10448 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10449 &GCPtrStack);
10450 if (RT_SUCCESS(rc))
10451 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10452 if (RT_FAILURE(rc))
10453 {
10454 rc = VERR_EM_INTERPRETER;
10455 break;
10456 }
10457 pMixedCtx->eip = 0;
10458 pMixedCtx->ip = aIretFrame[0];
10459 pMixedCtx->cs.Sel = aIretFrame[1];
10460 pMixedCtx->cs.ValidSel = aIretFrame[1];
10461 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10462 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10463 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10464 pMixedCtx->sp += sizeof(aIretFrame);
10465 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_SEGMENT_REGS | HM_CHANGED_GUEST_RSP
10466 | HM_CHANGED_GUEST_RFLAGS;
10467 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10468 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10469 break;
10470 }
10471
10472 case OP_INT:
10473 {
10474 uint16_t uVector = pDis->Param1.uValue & 0xff;
10475 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10476 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10477 break;
10478 }
10479
10480 case OP_INTO:
10481 {
10482 if (pMixedCtx->eflags.Bits.u1OF)
10483 {
10484 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10485 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10486 }
10487 break;
10488 }
10489
10490 default:
10491 {
10492 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10493 EMCODETYPE_SUPERVISOR);
10494 rc = VBOXSTRICTRC_VAL(rc2);
10495 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
10496 Log4(("#GP rc=%Rrc\n", rc));
10497 break;
10498 }
10499 }
10500 }
10501 else
10502 rc = VERR_EM_INTERPRETER;
10503
10504 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10505 ("#GP Unexpected rc=%Rrc\n", rc));
10506 return rc;
10507}
10508
10509
10510/**
10511 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10512 * the exception reported in the VMX transient structure back into the VM.
10513 *
10514 * @remarks Requires uExitIntrInfo in the VMX transient structure to be
10515 * up-to-date.
10516 */
10517static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10518{
10519 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10520
10521 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10522 hmR0VmxCheckExitDueToEventDelivery(). */
10523 int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10524 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10525 AssertRCReturn(rc, rc);
10526 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10527
10528 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10529 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10530 return VINF_SUCCESS;
10531}
10532
10533
10534/**
10535 * VM-exit exception handler for #PF (Page-fault exception).
10536 */
10537static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10538{
10539 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10540 PVM pVM = pVCpu->CTX_SUFF(pVM);
10541 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10542 rc |= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10543 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10544 AssertRCReturn(rc, rc);
10545
10546#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
10547 if (pVM->hm.s.fNestedPaging)
10548 {
10549 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10550 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10551 {
10552 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10553 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10554 0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10555 }
10556 else
10557 {
10558 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10559 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10560 Log4(("Pending #DF due to vectoring #PF. NP\n"));
10561 }
10562 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10563 return rc;
10564 }
10565#else
10566 Assert(!pVM->hm.s.fNestedPaging);
10567#endif
10568
10569 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10570 AssertRCReturn(rc, rc);
10571
10572 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10573 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10574
10575 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10576 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10577 (RTGCPTR)pVmxTransient->uExitQualification);
10578
10579 Log4(("#PF: rc=%Rrc\n", rc));
10580 if (rc == VINF_SUCCESS)
10581 {
10582 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10583 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
10584 * memory? We don't update the whole state here... */
10585 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10586 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10587 TRPMResetTrap(pVCpu);
10588 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10589 return rc;
10590 }
10591 else if (rc == VINF_EM_RAW_GUEST_TRAP)
10592 {
10593 if (!pVmxTransient->fVectoringPF)
10594 {
10595 /* It's a guest page fault and needs to be reflected to the guest. */
10596 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10597 TRPMResetTrap(pVCpu);
10598 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10599 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10600 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10601 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10602 }
10603 else
10604 {
10605 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10606 TRPMResetTrap(pVCpu);
10607 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10608 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10609 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10610 }
10611
10612 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10613 return VINF_SUCCESS;
10614 }
10615
10616 TRPMResetTrap(pVCpu);
10617 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10618 return rc;
10619}
10620
10621/** @} */
10622
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette