VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 54065

Last change on this file since 54065 was 54065, checked in by vboxsync, 10 years ago

VMM: Implemented TM TSC-mode switching with paravirtualized guests.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 506.1 KB
Line 
1/* $Id: HMVMXR0.cpp 54065 2015-02-03 10:45:39Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39# include "dtrace/VBoxVMM.h"
40
41#ifdef DEBUG_ramshankar
42# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
43# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
44# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_CHECK_GUEST_STATE
46# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
47# define HMVMX_ALWAYS_TRAP_PF
48# define HMVMX_ALWAYS_SWAP_FPU_STATE
49# define HMVMX_ALWAYS_FLUSH_TLB
50# define HMVMX_ALWAYS_SWAP_EFER
51#endif
52
53
54/*******************************************************************************
55* Defined Constants And Macros *
56*******************************************************************************/
57#if defined(RT_ARCH_AMD64)
58# define HMVMX_IS_64BIT_HOST_MODE() (true)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
61extern "C" uint32_t g_fVMXIs64bitHost;
62# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
63typedef uint64_t HMVMXHCUINTREG;
64#else
65# define HMVMX_IS_64BIT_HOST_MODE() (false)
66typedef RTHCUINTREG HMVMXHCUINTREG;
67#endif
68
69/** Use the function table. */
70#define HMVMX_USE_FUNCTION_TABLE
71
72/** Determine which tagged-TLB flush handler to use. */
73#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
74#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
75#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
76#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
77
78/** @name Updated-guest-state flags.
79 * @{ */
80#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
81#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
82#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
83#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
84#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
85#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
86#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
87#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
88#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
89#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
90#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
91#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
92#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
93#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
94#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
95#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
96#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
97#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
98#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
99#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
100#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
101 | HMVMX_UPDATED_GUEST_RSP \
102 | HMVMX_UPDATED_GUEST_RFLAGS \
103 | HMVMX_UPDATED_GUEST_CR0 \
104 | HMVMX_UPDATED_GUEST_CR3 \
105 | HMVMX_UPDATED_GUEST_CR4 \
106 | HMVMX_UPDATED_GUEST_GDTR \
107 | HMVMX_UPDATED_GUEST_IDTR \
108 | HMVMX_UPDATED_GUEST_LDTR \
109 | HMVMX_UPDATED_GUEST_TR \
110 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
111 | HMVMX_UPDATED_GUEST_DEBUG \
112 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
113 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
114 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
115 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
116 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
117 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
118 | HMVMX_UPDATED_GUEST_INTR_STATE \
119 | HMVMX_UPDATED_GUEST_APIC_STATE)
120/** @} */
121
122/** @name
123 * Flags to skip redundant reads of some common VMCS fields that are not part of
124 * the guest-CPU state but are in the transient structure.
125 */
126#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
127#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
131#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
132#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
133/** @} */
134
135/** @name
136 * States of the VMCS.
137 *
138 * This does not reflect all possible VMCS states but currently only those
139 * needed for maintaining the VMCS consistently even when thread-context hooks
140 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
141 */
142#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
143#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
144#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
145/** @} */
146
147/**
148 * Exception bitmap mask for real-mode guests (real-on-v86).
149 *
150 * We need to intercept all exceptions manually (except #PF). #NM is also
151 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
152 * even in real-mode if we have Nested Paging support.
153 */
154#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
155 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
156 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
157 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
158 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
159 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
160 | RT_BIT(X86_XCPT_XF))
161
162/**
163 * Exception bitmap mask for all contributory exceptions.
164 *
165 * Page fault is deliberately excluded here as it's conditional as to whether
166 * it's contributory or benign. Page faults are handled separately.
167 */
168#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
169 | RT_BIT(X86_XCPT_DE))
170
171/** Maximum VM-instruction error number. */
172#define HMVMX_INSTR_ERROR_MAX 28
173
174/** Profiling macro. */
175#ifdef HM_PROFILE_EXIT_DISPATCH
176# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
177# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
178#else
179# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
180# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
181#endif
182
183/** Assert that preemption is disabled or covered by thread-context hooks. */
184#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
185 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
186
187/** Assert that we haven't migrated CPUs when thread-context hooks are not
188 * used. */
189#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
190 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
191 ("Illegal migration! Entered on CPU %u Current %u\n", \
192 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
193
194/** Helper macro for VM-exit handlers called unexpectedly. */
195#define HMVMX_RETURN_UNEXPECTED_EXIT() \
196 do { \
197 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
198 return VERR_VMX_UNEXPECTED_EXIT; \
199 } while (0)
200
201
202/*******************************************************************************
203* Structures and Typedefs *
204*******************************************************************************/
205/**
206 * VMX transient state.
207 *
208 * A state structure for holding miscellaneous information across
209 * VMX non-root operation and restored after the transition.
210 */
211typedef struct VMXTRANSIENT
212{
213 /** The host's rflags/eflags. */
214 RTCCUINTREG uEflags;
215#if HC_ARCH_BITS == 32
216 uint32_t u32Alignment0;
217#endif
218 /** The guest's TPR value used for TPR shadowing. */
219 uint8_t u8GuestTpr;
220 /** Alignment. */
221 uint8_t abAlignment0[7];
222
223 /** The basic VM-exit reason. */
224 uint16_t uExitReason;
225 /** Alignment. */
226 uint16_t u16Alignment0;
227 /** The VM-exit interruption error code. */
228 uint32_t uExitIntErrorCode;
229 /** The VM-exit exit code qualification. */
230 uint64_t uExitQualification;
231
232 /** The VM-exit interruption-information field. */
233 uint32_t uExitIntInfo;
234 /** The VM-exit instruction-length field. */
235 uint32_t cbInstr;
236 /** The VM-exit instruction-information field. */
237 union
238 {
239 /** Plain unsigned int representation. */
240 uint32_t u;
241 /** INS and OUTS information. */
242 struct
243 {
244 uint32_t u6Reserved0 : 7;
245 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
246 uint32_t u3AddrSize : 3;
247 uint32_t u5Reserved1 : 5;
248 /** The segment register (X86_SREG_XXX). */
249 uint32_t iSegReg : 3;
250 uint32_t uReserved2 : 14;
251 } StrIo;
252 } ExitInstrInfo;
253 /** Whether the VM-entry failed or not. */
254 bool fVMEntryFailed;
255 /** Alignment. */
256 uint8_t abAlignment1[3];
257
258 /** The VM-entry interruption-information field. */
259 uint32_t uEntryIntInfo;
260 /** The VM-entry exception error code field. */
261 uint32_t uEntryXcptErrorCode;
262 /** The VM-entry instruction length field. */
263 uint32_t cbEntryInstr;
264
265 /** IDT-vectoring information field. */
266 uint32_t uIdtVectoringInfo;
267 /** IDT-vectoring error code. */
268 uint32_t uIdtVectoringErrorCode;
269
270 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
271 uint32_t fVmcsFieldsRead;
272
273 /** Whether the guest FPU was active at the time of VM-exit. */
274 bool fWasGuestFPUStateActive;
275 /** Whether the guest debug state was active at the time of VM-exit. */
276 bool fWasGuestDebugStateActive;
277 /** Whether the hyper debug state was active at the time of VM-exit. */
278 bool fWasHyperDebugStateActive;
279 /** Whether TSC-offsetting should be setup before VM-entry. */
280 bool fUpdateTscOffsettingAndPreemptTimer;
281 /** Whether the VM-exit was caused by a page-fault during delivery of a
282 * contributory exception or a page-fault. */
283 bool fVectoringDoublePF;
284 /** Whether the VM-exit was caused by a page-fault during delivery of an
285 * external interrupt or NMI. */
286 bool fVectoringPF;
287} VMXTRANSIENT;
288AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
289AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
290AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
291AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
292AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
293/** Pointer to VMX transient state. */
294typedef VMXTRANSIENT *PVMXTRANSIENT;
295
296
297/**
298 * MSR-bitmap read permissions.
299 */
300typedef enum VMXMSREXITREAD
301{
302 /** Reading this MSR causes a VM-exit. */
303 VMXMSREXIT_INTERCEPT_READ = 0xb,
304 /** Reading this MSR does not cause a VM-exit. */
305 VMXMSREXIT_PASSTHRU_READ
306} VMXMSREXITREAD;
307/** Pointer to MSR-bitmap read permissions. */
308typedef VMXMSREXITREAD* PVMXMSREXITREAD;
309
310/**
311 * MSR-bitmap write permissions.
312 */
313typedef enum VMXMSREXITWRITE
314{
315 /** Writing to this MSR causes a VM-exit. */
316 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
317 /** Writing to this MSR does not cause a VM-exit. */
318 VMXMSREXIT_PASSTHRU_WRITE
319} VMXMSREXITWRITE;
320/** Pointer to MSR-bitmap write permissions. */
321typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
322
323
324/**
325 * VMX VM-exit handler.
326 *
327 * @returns VBox status code.
328 * @param pVCpu Pointer to the VMCPU.
329 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
330 * out-of-sync. Make sure to update the required
331 * fields before using them.
332 * @param pVmxTransient Pointer to the VMX-transient structure.
333 */
334#ifndef HMVMX_USE_FUNCTION_TABLE
335typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
336#else
337typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
338/** Pointer to VM-exit handler. */
339typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
340#endif
341
342
343/*******************************************************************************
344* Internal Functions *
345*******************************************************************************/
346static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
347static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
348static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
349 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
350 bool fStepping, uint32_t *puIntState);
351#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
352static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
353#endif
354#ifndef HMVMX_USE_FUNCTION_TABLE
355DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
356# define HMVMX_EXIT_DECL static int
357#else
358# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
359#endif
360DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
361 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
362
363/** @name VM-exit handlers.
364 * @{
365 */
366static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
367static FNVMXEXITHANDLER hmR0VmxExitExtInt;
368static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
369static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
370static FNVMXEXITHANDLER hmR0VmxExitSipi;
371static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
372static FNVMXEXITHANDLER hmR0VmxExitSmi;
373static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
374static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
375static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
376static FNVMXEXITHANDLER hmR0VmxExitCpuid;
377static FNVMXEXITHANDLER hmR0VmxExitGetsec;
378static FNVMXEXITHANDLER hmR0VmxExitHlt;
379static FNVMXEXITHANDLER hmR0VmxExitInvd;
380static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
381static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
382static FNVMXEXITHANDLER hmR0VmxExitVmcall;
383static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
384static FNVMXEXITHANDLER hmR0VmxExitRsm;
385static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
386static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
387static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
388static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
389static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
390static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
391static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
392static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
393static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
394static FNVMXEXITHANDLER hmR0VmxExitMwait;
395static FNVMXEXITHANDLER hmR0VmxExitMtf;
396static FNVMXEXITHANDLER hmR0VmxExitMonitor;
397static FNVMXEXITHANDLER hmR0VmxExitPause;
398static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
399static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
400static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
401static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
402static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
403static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
404static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
405static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
406static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
407static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
408static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
409static FNVMXEXITHANDLER hmR0VmxExitRdrand;
410static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
411/** @} */
412
413static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
414static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
415static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
417static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
418static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
419#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
420static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
421#endif
422static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
423
424/*******************************************************************************
425* Global Variables *
426*******************************************************************************/
427#ifdef HMVMX_USE_FUNCTION_TABLE
428
429/**
430 * VMX_EXIT dispatch table.
431 */
432static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
433{
434 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
435 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
436 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
437 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
438 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
439 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
440 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
441 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
442 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
443 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
444 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
445 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
446 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
447 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
448 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
449 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
450 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
451 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
452 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
453 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
454 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
455 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
456 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
457 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
458 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
459 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
460 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
461 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
462 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
463 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
464 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
465 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
466 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
467 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
468 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
469 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
470 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
471 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
472 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
473 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
474 /* 40 UNDEFINED */ hmR0VmxExitPause,
475 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
476 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
477 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
478 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
479 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
480 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
481 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
482 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
483 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
484 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
485 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
486 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
487 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
488 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
489 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
490 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
491 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
492 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
493 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
494};
495#endif /* HMVMX_USE_FUNCTION_TABLE */
496
497#ifdef VBOX_STRICT
498static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
499{
500 /* 0 */ "(Not Used)",
501 /* 1 */ "VMCALL executed in VMX root operation.",
502 /* 2 */ "VMCLEAR with invalid physical address.",
503 /* 3 */ "VMCLEAR with VMXON pointer.",
504 /* 4 */ "VMLAUNCH with non-clear VMCS.",
505 /* 5 */ "VMRESUME with non-launched VMCS.",
506 /* 6 */ "VMRESUME after VMXOFF",
507 /* 7 */ "VM-entry with invalid control fields.",
508 /* 8 */ "VM-entry with invalid host state fields.",
509 /* 9 */ "VMPTRLD with invalid physical address.",
510 /* 10 */ "VMPTRLD with VMXON pointer.",
511 /* 11 */ "VMPTRLD with incorrect revision identifier.",
512 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
513 /* 13 */ "VMWRITE to read-only VMCS component.",
514 /* 14 */ "(Not Used)",
515 /* 15 */ "VMXON executed in VMX root operation.",
516 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
517 /* 17 */ "VM-entry with non-launched executing VMCS.",
518 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
519 /* 19 */ "VMCALL with non-clear VMCS.",
520 /* 20 */ "VMCALL with invalid VM-exit control fields.",
521 /* 21 */ "(Not Used)",
522 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
523 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
524 /* 24 */ "VMCALL with invalid SMM-monitor features.",
525 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
526 /* 26 */ "VM-entry with events blocked by MOV SS.",
527 /* 27 */ "(Not Used)",
528 /* 28 */ "Invalid operand to INVEPT/INVVPID."
529};
530#endif /* VBOX_STRICT */
531
532
533
534/**
535 * Updates the VM's last error record. If there was a VMX instruction error,
536 * reads the error data from the VMCS and updates VCPU's last error record as
537 * well.
538 *
539 * @param pVM Pointer to the VM.
540 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
541 * VERR_VMX_UNABLE_TO_START_VM or
542 * VERR_VMX_INVALID_VMCS_FIELD).
543 * @param rc The error code.
544 */
545static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
546{
547 AssertPtr(pVM);
548 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
549 || rc == VERR_VMX_UNABLE_TO_START_VM)
550 {
551 AssertPtrReturnVoid(pVCpu);
552 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
553 }
554 pVM->hm.s.lLastError = rc;
555}
556
557
558/**
559 * Reads the VM-entry interruption-information field from the VMCS into the VMX
560 * transient structure.
561 *
562 * @returns VBox status code.
563 * @param pVmxTransient Pointer to the VMX transient structure.
564 *
565 * @remarks No-long-jump zone!!!
566 */
567DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
568{
569 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
570 AssertRCReturn(rc, rc);
571 return VINF_SUCCESS;
572}
573
574
575/**
576 * Reads the VM-entry exception error code field from the VMCS into
577 * the VMX transient structure.
578 *
579 * @returns VBox status code.
580 * @param pVmxTransient Pointer to the VMX transient structure.
581 *
582 * @remarks No-long-jump zone!!!
583 */
584DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
585{
586 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
587 AssertRCReturn(rc, rc);
588 return VINF_SUCCESS;
589}
590
591
592/**
593 * Reads the VM-entry exception error code field from the VMCS into
594 * the VMX transient structure.
595 *
596 * @returns VBox status code.
597 * @param pVmxTransient Pointer to the VMX transient structure.
598 *
599 * @remarks No-long-jump zone!!!
600 */
601DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
602{
603 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
604 AssertRCReturn(rc, rc);
605 return VINF_SUCCESS;
606}
607
608
609/**
610 * Reads the VM-exit interruption-information field from the VMCS into the VMX
611 * transient structure.
612 *
613 * @returns VBox status code.
614 * @param pVmxTransient Pointer to the VMX transient structure.
615 */
616DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
617{
618 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
619 {
620 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
621 AssertRCReturn(rc, rc);
622 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
623 }
624 return VINF_SUCCESS;
625}
626
627
628/**
629 * Reads the VM-exit interruption error code from the VMCS into the VMX
630 * transient structure.
631 *
632 * @returns VBox status code.
633 * @param pVmxTransient Pointer to the VMX transient structure.
634 */
635DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
636{
637 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
638 {
639 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
640 AssertRCReturn(rc, rc);
641 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
642 }
643 return VINF_SUCCESS;
644}
645
646
647/**
648 * Reads the VM-exit instruction length field from the VMCS into the VMX
649 * transient structure.
650 *
651 * @returns VBox status code.
652 * @param pVCpu Pointer to the VMCPU.
653 * @param pVmxTransient Pointer to the VMX transient structure.
654 */
655DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
656{
657 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
658 {
659 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
660 AssertRCReturn(rc, rc);
661 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
662 }
663 return VINF_SUCCESS;
664}
665
666
667/**
668 * Reads the VM-exit instruction-information field from the VMCS into
669 * the VMX transient structure.
670 *
671 * @returns VBox status code.
672 * @param pVmxTransient Pointer to the VMX transient structure.
673 */
674DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
675{
676 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
677 {
678 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
679 AssertRCReturn(rc, rc);
680 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
681 }
682 return VINF_SUCCESS;
683}
684
685
686/**
687 * Reads the exit code qualification from the VMCS into the VMX transient
688 * structure.
689 *
690 * @returns VBox status code.
691 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
692 * case).
693 * @param pVmxTransient Pointer to the VMX transient structure.
694 */
695DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
696{
697 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
698 {
699 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
700 AssertRCReturn(rc, rc);
701 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
702 }
703 return VINF_SUCCESS;
704}
705
706
707/**
708 * Reads the IDT-vectoring information field from the VMCS into the VMX
709 * transient structure.
710 *
711 * @returns VBox status code.
712 * @param pVmxTransient Pointer to the VMX transient structure.
713 *
714 * @remarks No-long-jump zone!!!
715 */
716DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
717{
718 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
719 {
720 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
721 AssertRCReturn(rc, rc);
722 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
723 }
724 return VINF_SUCCESS;
725}
726
727
728/**
729 * Reads the IDT-vectoring error code from the VMCS into the VMX
730 * transient structure.
731 *
732 * @returns VBox status code.
733 * @param pVmxTransient Pointer to the VMX transient structure.
734 */
735DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
736{
737 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
738 {
739 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
740 AssertRCReturn(rc, rc);
741 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
742 }
743 return VINF_SUCCESS;
744}
745
746
747/**
748 * Enters VMX root mode operation on the current CPU.
749 *
750 * @returns VBox status code.
751 * @param pVM Pointer to the VM (optional, can be NULL, after
752 * a resume).
753 * @param HCPhysCpuPage Physical address of the VMXON region.
754 * @param pvCpuPage Pointer to the VMXON region.
755 */
756static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
757{
758 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
759 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
760 Assert(pvCpuPage);
761 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
762
763 if (pVM)
764 {
765 /* Write the VMCS revision dword to the VMXON region. */
766 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
767 }
768
769 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
770 RTCCUINTREG uEflags = ASMIntDisableFlags();
771
772 /* Enable the VMX bit in CR4 if necessary. */
773 RTCCUINTREG uCr4 = ASMGetCR4();
774 if (!(uCr4 & X86_CR4_VMXE))
775 ASMSetCR4(uCr4 | X86_CR4_VMXE);
776
777 /* Enter VMX root mode. */
778 int rc = VMXEnable(HCPhysCpuPage);
779 if (RT_FAILURE(rc))
780 ASMSetCR4(uCr4);
781
782 /* Restore interrupts. */
783 ASMSetFlags(uEflags);
784 return rc;
785}
786
787
788/**
789 * Exits VMX root mode operation on the current CPU.
790 *
791 * @returns VBox status code.
792 */
793static int hmR0VmxLeaveRootMode(void)
794{
795 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
796
797 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
798 RTCCUINTREG uEflags = ASMIntDisableFlags();
799
800 /* If we're for some reason not in VMX root mode, then don't leave it. */
801 RTCCUINTREG uHostCR4 = ASMGetCR4();
802
803 int rc;
804 if (uHostCR4 & X86_CR4_VMXE)
805 {
806 /* Exit VMX root mode and clear the VMX bit in CR4. */
807 VMXDisable();
808 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
809 rc = VINF_SUCCESS;
810 }
811 else
812 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
813
814 /* Restore interrupts. */
815 ASMSetFlags(uEflags);
816 return rc;
817}
818
819
820/**
821 * Allocates and maps one physically contiguous page. The allocated page is
822 * zero'd out. (Used by various VT-x structures).
823 *
824 * @returns IPRT status code.
825 * @param pMemObj Pointer to the ring-0 memory object.
826 * @param ppVirt Where to store the virtual address of the
827 * allocation.
828 * @param pPhys Where to store the physical address of the
829 * allocation.
830 */
831DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
832{
833 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
834 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
835 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
836
837 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
838 if (RT_FAILURE(rc))
839 return rc;
840 *ppVirt = RTR0MemObjAddress(*pMemObj);
841 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
842 ASMMemZero32(*ppVirt, PAGE_SIZE);
843 return VINF_SUCCESS;
844}
845
846
847/**
848 * Frees and unmaps an allocated physical page.
849 *
850 * @param pMemObj Pointer to the ring-0 memory object.
851 * @param ppVirt Where to re-initialize the virtual address of
852 * allocation as 0.
853 * @param pHCPhys Where to re-initialize the physical address of the
854 * allocation as 0.
855 */
856DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
857{
858 AssertPtr(pMemObj);
859 AssertPtr(ppVirt);
860 AssertPtr(pHCPhys);
861 if (*pMemObj != NIL_RTR0MEMOBJ)
862 {
863 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
864 AssertRC(rc);
865 *pMemObj = NIL_RTR0MEMOBJ;
866 *ppVirt = 0;
867 *pHCPhys = 0;
868 }
869}
870
871
872/**
873 * Worker function to free VT-x related structures.
874 *
875 * @returns IPRT status code.
876 * @param pVM Pointer to the VM.
877 */
878static void hmR0VmxStructsFree(PVM pVM)
879{
880 for (VMCPUID i = 0; i < pVM->cCpus; i++)
881 {
882 PVMCPU pVCpu = &pVM->aCpus[i];
883 AssertPtr(pVCpu);
884
885 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
886 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
887
888 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
889 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
890
891 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
892 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
893 }
894
895 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
896#ifdef VBOX_WITH_CRASHDUMP_MAGIC
897 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
898#endif
899}
900
901
902/**
903 * Worker function to allocate VT-x related VM structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM Pointer to the VM.
907 */
908static int hmR0VmxStructsAlloc(PVM pVM)
909{
910 /*
911 * Initialize members up-front so we can cleanup properly on allocation failure.
912 */
913#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
914 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
915 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
916 pVM->hm.s.vmx.HCPhys##a_Name = 0;
917
918#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
919 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
920 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
921 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
922
923#ifdef VBOX_WITH_CRASHDUMP_MAGIC
924 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
925#endif
926 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
927
928 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
929 for (VMCPUID i = 0; i < pVM->cCpus; i++)
930 {
931 PVMCPU pVCpu = &pVM->aCpus[i];
932 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
935 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
936 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
937 }
938#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
939#undef VMXLOCAL_INIT_VM_MEMOBJ
940
941 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
942 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
943 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
944 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
945
946 /*
947 * Allocate all the VT-x structures.
948 */
949 int rc = VINF_SUCCESS;
950#ifdef VBOX_WITH_CRASHDUMP_MAGIC
951 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
952 if (RT_FAILURE(rc))
953 goto cleanup;
954 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
955 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
956#endif
957
958 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
959 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
960 {
961 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
962 &pVM->hm.s.vmx.HCPhysApicAccess);
963 if (RT_FAILURE(rc))
964 goto cleanup;
965 }
966
967 /*
968 * Initialize per-VCPU VT-x structures.
969 */
970 for (VMCPUID i = 0; i < pVM->cCpus; i++)
971 {
972 PVMCPU pVCpu = &pVM->aCpus[i];
973 AssertPtr(pVCpu);
974
975 /* Allocate the VM control structure (VMCS). */
976 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
977 if (RT_FAILURE(rc))
978 goto cleanup;
979
980 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
981 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
982 {
983 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
984 &pVCpu->hm.s.vmx.HCPhysVirtApic);
985 if (RT_FAILURE(rc))
986 goto cleanup;
987 }
988
989 /*
990 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
991 * transparent accesses of specific MSRs.
992 *
993 * If the condition for enabling MSR bitmaps changes here, don't forget to
994 * update HMIsMsrBitmapsAvailable().
995 */
996 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
997 {
998 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
999 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1000 if (RT_FAILURE(rc))
1001 goto cleanup;
1002 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1003 }
1004
1005 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1006 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1007 if (RT_FAILURE(rc))
1008 goto cleanup;
1009
1010 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1012 if (RT_FAILURE(rc))
1013 goto cleanup;
1014 }
1015
1016 return VINF_SUCCESS;
1017
1018cleanup:
1019 hmR0VmxStructsFree(pVM);
1020 return rc;
1021}
1022
1023
1024/**
1025 * Does global VT-x initialization (called during module initialization).
1026 *
1027 * @returns VBox status code.
1028 */
1029VMMR0DECL(int) VMXR0GlobalInit(void)
1030{
1031#ifdef HMVMX_USE_FUNCTION_TABLE
1032 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1033# ifdef VBOX_STRICT
1034 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1035 Assert(g_apfnVMExitHandlers[i]);
1036# endif
1037#endif
1038 return VINF_SUCCESS;
1039}
1040
1041
1042/**
1043 * Does global VT-x termination (called during module termination).
1044 */
1045VMMR0DECL(void) VMXR0GlobalTerm()
1046{
1047 /* Nothing to do currently. */
1048}
1049
1050
1051/**
1052 * Sets up and activates VT-x on the current CPU.
1053 *
1054 * @returns VBox status code.
1055 * @param pCpu Pointer to the global CPU info struct.
1056 * @param pVM Pointer to the VM (can be NULL after a host resume
1057 * operation).
1058 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1059 * fEnabledByHost is true).
1060 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1061 * @a fEnabledByHost is true).
1062 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1063 * enable VT-x on the host.
1064 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1065 */
1066VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1067 void *pvMsrs)
1068{
1069 Assert(pCpu);
1070 Assert(pvMsrs);
1071 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1072
1073 /* Enable VT-x if it's not already enabled by the host. */
1074 if (!fEnabledByHost)
1075 {
1076 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1077 if (RT_FAILURE(rc))
1078 return rc;
1079 }
1080
1081 /*
1082 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1083 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1084 */
1085 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1086 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1087 {
1088 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1089 pCpu->fFlushAsidBeforeUse = false;
1090 }
1091 else
1092 pCpu->fFlushAsidBeforeUse = true;
1093
1094 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1095 ++pCpu->cTlbFlushes;
1096
1097 return VINF_SUCCESS;
1098}
1099
1100
1101/**
1102 * Deactivates VT-x on the current CPU.
1103 *
1104 * @returns VBox status code.
1105 * @param pCpu Pointer to the global CPU info struct.
1106 * @param pvCpuPage Pointer to the VMXON region.
1107 * @param HCPhysCpuPage Physical address of the VMXON region.
1108 *
1109 * @remarks This function should never be called when SUPR0EnableVTx() or
1110 * similar was used to enable VT-x on the host.
1111 */
1112VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1113{
1114 NOREF(pCpu);
1115 NOREF(pvCpuPage);
1116 NOREF(HCPhysCpuPage);
1117
1118 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1119 return hmR0VmxLeaveRootMode();
1120}
1121
1122
1123/**
1124 * Sets the permission bits for the specified MSR in the MSR bitmap.
1125 *
1126 * @param pVCpu Pointer to the VMCPU.
1127 * @param uMSR The MSR value.
1128 * @param enmRead Whether reading this MSR causes a VM-exit.
1129 * @param enmWrite Whether writing this MSR causes a VM-exit.
1130 */
1131static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1132{
1133 int32_t iBit;
1134 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1135
1136 /*
1137 * Layout:
1138 * 0x000 - 0x3ff - Low MSR read bits
1139 * 0x400 - 0x7ff - High MSR read bits
1140 * 0x800 - 0xbff - Low MSR write bits
1141 * 0xc00 - 0xfff - High MSR write bits
1142 */
1143 if (uMsr <= 0x00001FFF)
1144 iBit = uMsr;
1145 else if ( uMsr >= 0xC0000000
1146 && uMsr <= 0xC0001FFF)
1147 {
1148 iBit = (uMsr - 0xC0000000);
1149 pbMsrBitmap += 0x400;
1150 }
1151 else
1152 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1153
1154 Assert(iBit <= 0x1fff);
1155 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1156 ASMBitSet(pbMsrBitmap, iBit);
1157 else
1158 ASMBitClear(pbMsrBitmap, iBit);
1159
1160 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1161 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1162 else
1163 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1164}
1165
1166
1167#ifdef VBOX_STRICT
1168/**
1169 * Gets the permission bits for the specified MSR in the MSR bitmap.
1170 *
1171 * @returns VBox status code.
1172 * @retval VINF_SUCCESS if the specified MSR is found.
1173 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1174 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1175 *
1176 * @param pVCpu Pointer to the VMCPU.
1177 * @param uMsr The MSR.
1178 * @param penmRead Where to store the read permissions.
1179 * @param penmWrite Where to store the write permissions.
1180 */
1181static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1182{
1183 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1184 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1185 int32_t iBit;
1186 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1187
1188 /* See hmR0VmxSetMsrPermission() for the layout. */
1189 if (uMsr <= 0x00001FFF)
1190 iBit = uMsr;
1191 else if ( uMsr >= 0xC0000000
1192 && uMsr <= 0xC0001FFF)
1193 {
1194 iBit = (uMsr - 0xC0000000);
1195 pbMsrBitmap += 0x400;
1196 }
1197 else
1198 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1199
1200 Assert(iBit <= 0x1fff);
1201 if (ASMBitTest(pbMsrBitmap, iBit))
1202 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1203 else
1204 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1205
1206 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1207 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1208 else
1209 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1210 return VINF_SUCCESS;
1211}
1212#endif /* VBOX_STRICT */
1213
1214
1215/**
1216 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1217 * area.
1218 *
1219 * @returns VBox status code.
1220 * @param pVCpu Pointer to the VMCPU.
1221 * @param cMsrs The number of MSRs.
1222 */
1223DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1224{
1225 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1226 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1227 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1228 {
1229 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1230 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1231 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1232 }
1233
1234 /* Update number of guest MSRs to load/store across the world-switch. */
1235 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1236 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1237
1238 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1239 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1240
1241 /* Update the VCPU's copy of the MSR count. */
1242 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1243
1244 return VINF_SUCCESS;
1245}
1246
1247
1248/**
1249 * Adds a new (or updates the value of an existing) guest/host MSR
1250 * pair to be swapped during the world-switch as part of the
1251 * auto-load/store MSR area in the VMCS.
1252 *
1253 * @returns true if the MSR was added -and- its value was updated, false
1254 * otherwise.
1255 * @param pVCpu Pointer to the VMCPU.
1256 * @param uMsr The MSR.
1257 * @param uGuestMsr Value of the guest MSR.
1258 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1259 * necessary.
1260 */
1261static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1262{
1263 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1264 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1265 uint32_t i;
1266 for (i = 0; i < cMsrs; i++)
1267 {
1268 if (pGuestMsr->u32Msr == uMsr)
1269 break;
1270 pGuestMsr++;
1271 }
1272
1273 bool fAdded = false;
1274 if (i == cMsrs)
1275 {
1276 ++cMsrs;
1277 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1278 AssertRC(rc);
1279
1280 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1281 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1282 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1283
1284 fAdded = true;
1285 }
1286
1287 /* Update the MSR values in the auto-load/store MSR area. */
1288 pGuestMsr->u32Msr = uMsr;
1289 pGuestMsr->u64Value = uGuestMsrValue;
1290
1291 /* Create/update the MSR slot in the host MSR area. */
1292 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1293 pHostMsr += i;
1294 pHostMsr->u32Msr = uMsr;
1295
1296 /*
1297 * Update the host MSR only when requested by the caller AND when we're
1298 * adding it to the auto-load/store area. Otherwise, it would have been
1299 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1300 */
1301 bool fUpdatedMsrValue = false;
1302 if ( fAdded
1303 && fUpdateHostMsr)
1304 {
1305 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1306 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1307 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1308 fUpdatedMsrValue = true;
1309 }
1310
1311 return fUpdatedMsrValue;
1312}
1313
1314
1315/**
1316 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1317 * auto-load/store MSR area in the VMCS.
1318 *
1319 * @returns VBox status code.
1320 * @param pVCpu Pointer to the VMCPU.
1321 * @param uMsr The MSR.
1322 */
1323static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1324{
1325 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1326 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1327 for (uint32_t i = 0; i < cMsrs; i++)
1328 {
1329 /* Find the MSR. */
1330 if (pGuestMsr->u32Msr == uMsr)
1331 {
1332 /* If it's the last MSR, simply reduce the count. */
1333 if (i == cMsrs - 1)
1334 {
1335 --cMsrs;
1336 break;
1337 }
1338
1339 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1340 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1341 pLastGuestMsr += cMsrs - 1;
1342 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1343 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1344
1345 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1346 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1347 pLastHostMsr += cMsrs - 1;
1348 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1349 pHostMsr->u64Value = pLastHostMsr->u64Value;
1350 --cMsrs;
1351 break;
1352 }
1353 pGuestMsr++;
1354 }
1355
1356 /* Update the VMCS if the count changed (meaning the MSR was found). */
1357 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1358 {
1359 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1360 AssertRCReturn(rc, rc);
1361
1362 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1363 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1364 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1365
1366 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1367 return VINF_SUCCESS;
1368 }
1369
1370 return VERR_NOT_FOUND;
1371}
1372
1373
1374/**
1375 * Checks if the specified guest MSR is part of the auto-load/store area in
1376 * the VMCS.
1377 *
1378 * @returns true if found, false otherwise.
1379 * @param pVCpu Pointer to the VMCPU.
1380 * @param uMsr The MSR to find.
1381 */
1382static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1383{
1384 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1385 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1386
1387 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1388 {
1389 if (pGuestMsr->u32Msr == uMsr)
1390 return true;
1391 }
1392 return false;
1393}
1394
1395
1396/**
1397 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1398 *
1399 * @param pVCpu Pointer to the VMCPU.
1400 *
1401 * @remarks No-long-jump zone!!!
1402 */
1403static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1404{
1405 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1406 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1407 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1408 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1409
1410 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1411 {
1412 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1413
1414 /*
1415 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1416 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1417 */
1418 if (pHostMsr->u32Msr == MSR_K6_EFER)
1419 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1420 else
1421 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1422 }
1423
1424 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1425}
1426
1427
1428#if HC_ARCH_BITS == 64
1429/**
1430 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1431 * perform lazy restoration of the host MSRs while leaving VT-x.
1432 *
1433 * @param pVCpu Pointer to the VMCPU.
1434 *
1435 * @remarks No-long-jump zone!!!
1436 */
1437static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1438{
1439 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1440
1441 /*
1442 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1443 */
1444 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1445 {
1446 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1447 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1448 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1449 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1450 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1451 }
1452}
1453
1454
1455/**
1456 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1457 * lazily while leaving VT-x.
1458 *
1459 * @returns true if it does, false otherwise.
1460 * @param pVCpu Pointer to the VMCPU.
1461 * @param uMsr The MSR to check.
1462 */
1463static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1464{
1465 NOREF(pVCpu);
1466 switch (uMsr)
1467 {
1468 case MSR_K8_LSTAR:
1469 case MSR_K6_STAR:
1470 case MSR_K8_SF_MASK:
1471 case MSR_K8_KERNEL_GS_BASE:
1472 return true;
1473 }
1474 return false;
1475}
1476
1477
1478/**
1479 * Saves a set of guest MSRs back into the guest-CPU context.
1480 *
1481 * @param pVCpu Pointer to the VMCPU.
1482 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1483 * out-of-sync. Make sure to update the required fields
1484 * before using them.
1485 *
1486 * @remarks No-long-jump zone!!!
1487 */
1488static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1489{
1490 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1491 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1492
1493 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1494 {
1495 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1496 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1497 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1498 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1499 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1500 }
1501}
1502
1503
1504/**
1505 * Loads a set of guests MSRs to allow read/passthru to the guest.
1506 *
1507 * The name of this function is slightly confusing. This function does NOT
1508 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1509 * common prefix for functions dealing with "lazy restoration" of the shared
1510 * MSRs.
1511 *
1512 * @param pVCpu Pointer to the VMCPU.
1513 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1514 * out-of-sync. Make sure to update the required fields
1515 * before using them.
1516 *
1517 * @remarks No-long-jump zone!!!
1518 */
1519static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1520{
1521 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1522 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1523
1524#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1525 do { \
1526 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1527 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1528 else \
1529 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1530 } while (0)
1531
1532 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1533 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1534 {
1535 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1536 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1537 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1538 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1539 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1540 }
1541 else
1542 {
1543 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1544 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1545 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1546 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1547 }
1548
1549#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1550}
1551
1552
1553/**
1554 * Performs lazy restoration of the set of host MSRs if they were previously
1555 * loaded with guest MSR values.
1556 *
1557 * @param pVCpu Pointer to the VMCPU.
1558 *
1559 * @remarks No-long-jump zone!!!
1560 * @remarks The guest MSRs should have been saved back into the guest-CPU
1561 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1562 */
1563static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1564{
1565 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1566 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1567
1568 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1569 {
1570 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1571 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1572 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1573 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1574 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1575 }
1576 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1577}
1578#endif /* HC_ARCH_BITS == 64 */
1579
1580
1581/**
1582 * Verifies that our cached values of the VMCS controls are all
1583 * consistent with what's actually present in the VMCS.
1584 *
1585 * @returns VBox status code.
1586 * @param pVCpu Pointer to the VMCPU.
1587 */
1588static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1589{
1590 uint32_t u32Val;
1591 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1592 AssertRCReturn(rc, rc);
1593 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1594 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1595
1596 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1597 AssertRCReturn(rc, rc);
1598 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1599 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1600
1601 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1602 AssertRCReturn(rc, rc);
1603 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1604 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1605
1606 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1607 AssertRCReturn(rc, rc);
1608 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1609 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1610
1611 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1612 {
1613 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1614 AssertRCReturn(rc, rc);
1615 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1616 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1617 }
1618
1619 return VINF_SUCCESS;
1620}
1621
1622
1623#ifdef VBOX_STRICT
1624/**
1625 * Verifies that our cached host EFER value has not changed
1626 * since we cached it.
1627 *
1628 * @param pVCpu Pointer to the VMCPU.
1629 */
1630static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1631{
1632 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1633
1634 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1635 {
1636 uint64_t u64Val;
1637 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1638 AssertRC(rc);
1639
1640 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1641 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1642 }
1643}
1644
1645
1646/**
1647 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1648 * VMCS are correct.
1649 *
1650 * @param pVCpu Pointer to the VMCPU.
1651 */
1652static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1653{
1654 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1655
1656 /* Verify MSR counts in the VMCS are what we think it should be. */
1657 uint32_t cMsrs;
1658 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1659 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1660
1661 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1662 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1663
1664 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1665 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1666
1667 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1668 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1669 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1670 {
1671 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1672 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1673 pGuestMsr->u32Msr, cMsrs));
1674
1675 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1676 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1677 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1678
1679 /* Verify that the permissions are as expected in the MSR bitmap. */
1680 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1681 {
1682 VMXMSREXITREAD enmRead;
1683 VMXMSREXITWRITE enmWrite;
1684 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1685 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1686 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1687 {
1688 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1689 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1690 }
1691 else
1692 {
1693 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1694 pGuestMsr->u32Msr, cMsrs));
1695 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1696 pGuestMsr->u32Msr, cMsrs));
1697 }
1698 }
1699 }
1700}
1701#endif /* VBOX_STRICT */
1702
1703
1704/**
1705 * Flushes the TLB using EPT.
1706 *
1707 * @returns VBox status code.
1708 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1709 * enmFlush).
1710 * @param enmFlush Type of flush.
1711 *
1712 * @remarks Caller is responsible for making sure this function is called only
1713 * when NestedPaging is supported and providing @a enmFlush that is
1714 * supported by the CPU.
1715 * @remarks Can be called with interrupts disabled.
1716 */
1717static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1718{
1719 uint64_t au64Descriptor[2];
1720 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1721 au64Descriptor[0] = 0;
1722 else
1723 {
1724 Assert(pVCpu);
1725 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1726 }
1727 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1728
1729 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1730 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1731 rc));
1732 if ( RT_SUCCESS(rc)
1733 && pVCpu)
1734 {
1735 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1736 }
1737}
1738
1739
1740/**
1741 * Flushes the TLB using VPID.
1742 *
1743 * @returns VBox status code.
1744 * @param pVM Pointer to the VM.
1745 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1746 * enmFlush).
1747 * @param enmFlush Type of flush.
1748 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1749 * on @a enmFlush).
1750 *
1751 * @remarks Can be called with interrupts disabled.
1752 */
1753static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1754{
1755 NOREF(pVM);
1756 AssertPtr(pVM);
1757 Assert(pVM->hm.s.vmx.fVpid);
1758
1759 uint64_t au64Descriptor[2];
1760 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1761 {
1762 au64Descriptor[0] = 0;
1763 au64Descriptor[1] = 0;
1764 }
1765 else
1766 {
1767 AssertPtr(pVCpu);
1768 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1769 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1770 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1771 au64Descriptor[1] = GCPtr;
1772 }
1773
1774 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1775 AssertMsg(rc == VINF_SUCCESS,
1776 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1777 if ( RT_SUCCESS(rc)
1778 && pVCpu)
1779 {
1780 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1781 }
1782}
1783
1784
1785/**
1786 * Invalidates a guest page by guest virtual address. Only relevant for
1787 * EPT/VPID, otherwise there is nothing really to invalidate.
1788 *
1789 * @returns VBox status code.
1790 * @param pVM Pointer to the VM.
1791 * @param pVCpu Pointer to the VMCPU.
1792 * @param GCVirt Guest virtual address of the page to invalidate.
1793 */
1794VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1795{
1796 AssertPtr(pVM);
1797 AssertPtr(pVCpu);
1798 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1799
1800 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1801 if (!fFlushPending)
1802 {
1803 /*
1804 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1805 * See @bugref{6043} and @bugref{6177}.
1806 *
1807 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1808 * function maybe called in a loop with individual addresses.
1809 */
1810 if (pVM->hm.s.vmx.fVpid)
1811 {
1812 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1813 {
1814 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1815 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1816 }
1817 else
1818 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1819 }
1820 else if (pVM->hm.s.fNestedPaging)
1821 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1822 }
1823
1824 return VINF_SUCCESS;
1825}
1826
1827
1828/**
1829 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1830 * otherwise there is nothing really to invalidate.
1831 *
1832 * @returns VBox status code.
1833 * @param pVM Pointer to the VM.
1834 * @param pVCpu Pointer to the VMCPU.
1835 * @param GCPhys Guest physical address of the page to invalidate.
1836 */
1837VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1838{
1839 NOREF(pVM); NOREF(GCPhys);
1840 LogFlowFunc(("%RGp\n", GCPhys));
1841
1842 /*
1843 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1844 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1845 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1846 */
1847 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1848 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1849 return VINF_SUCCESS;
1850}
1851
1852
1853/**
1854 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1855 * case where neither EPT nor VPID is supported by the CPU.
1856 *
1857 * @param pVM Pointer to the VM.
1858 * @param pVCpu Pointer to the VMCPU.
1859 * @param pCpu Pointer to the global HM struct.
1860 *
1861 * @remarks Called with interrupts disabled.
1862 */
1863static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1864{
1865 AssertPtr(pVCpu);
1866 AssertPtr(pCpu);
1867 NOREF(pVM);
1868
1869 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1870
1871 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1872#if 0
1873 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1874 pVCpu->hm.s.TlbShootdown.cPages = 0;
1875#endif
1876
1877 Assert(pCpu->idCpu != NIL_RTCPUID);
1878 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1879 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1880 pVCpu->hm.s.fForceTLBFlush = false;
1881 return;
1882}
1883
1884
1885/**
1886 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1887 *
1888 * @param pVM Pointer to the VM.
1889 * @param pVCpu Pointer to the VMCPU.
1890 * @param pCpu Pointer to the global HM CPU struct.
1891 * @remarks All references to "ASID" in this function pertains to "VPID" in
1892 * Intel's nomenclature. The reason is, to avoid confusion in compare
1893 * statements since the host-CPU copies are named "ASID".
1894 *
1895 * @remarks Called with interrupts disabled.
1896 */
1897static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1898{
1899#ifdef VBOX_WITH_STATISTICS
1900 bool fTlbFlushed = false;
1901# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1902# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1903 if (!fTlbFlushed) \
1904 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1905 } while (0)
1906#else
1907# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1908# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1909#endif
1910
1911 AssertPtr(pVM);
1912 AssertPtr(pCpu);
1913 AssertPtr(pVCpu);
1914 Assert(pCpu->idCpu != NIL_RTCPUID);
1915
1916 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1917 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1918 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1919
1920 /*
1921 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1922 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1923 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1924 */
1925 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1926 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1927 {
1928 ++pCpu->uCurrentAsid;
1929 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1930 {
1931 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1932 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1933 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1934 }
1935
1936 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1937 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1938 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1939
1940 /*
1941 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1942 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1943 */
1944 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1945 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1946 HMVMX_SET_TAGGED_TLB_FLUSHED();
1947 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1948 }
1949
1950 /* Check for explicit TLB shootdowns. */
1951 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1952 {
1953 /*
1954 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1955 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1956 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1957 * but not guest-physical mappings.
1958 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1959 */
1960 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1961 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1962 HMVMX_SET_TAGGED_TLB_FLUSHED();
1963 }
1964
1965 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1966 * where it is commented out. Support individual entry flushing
1967 * someday. */
1968#if 0
1969 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1970 {
1971 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1972
1973 /*
1974 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1975 * as supported by the CPU.
1976 */
1977 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1978 {
1979 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1980 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1981 }
1982 else
1983 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1984
1985 HMVMX_SET_TAGGED_TLB_FLUSHED();
1986 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1987 pVCpu->hm.s.TlbShootdown.cPages = 0;
1988 }
1989#endif
1990
1991 pVCpu->hm.s.fForceTLBFlush = false;
1992
1993 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1994
1995 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1996 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1997 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1998 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1999 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2000 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2001 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2002 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2003 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2004
2005 /* Update VMCS with the VPID. */
2006 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2007 AssertRC(rc);
2008
2009#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2010}
2011
2012
2013/**
2014 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2015 *
2016 * @returns VBox status code.
2017 * @param pVM Pointer to the VM.
2018 * @param pVCpu Pointer to the VMCPU.
2019 * @param pCpu Pointer to the global HM CPU struct.
2020 *
2021 * @remarks Called with interrupts disabled.
2022 */
2023static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2024{
2025 AssertPtr(pVM);
2026 AssertPtr(pVCpu);
2027 AssertPtr(pCpu);
2028 Assert(pCpu->idCpu != NIL_RTCPUID);
2029 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2030 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2031
2032 /*
2033 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2034 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2035 */
2036 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2037 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2038 {
2039 pVCpu->hm.s.fForceTLBFlush = true;
2040 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2041 }
2042
2043 /* Check for explicit TLB shootdown flushes. */
2044 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2045 {
2046 pVCpu->hm.s.fForceTLBFlush = true;
2047 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2048 }
2049
2050 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2051 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2052
2053 if (pVCpu->hm.s.fForceTLBFlush)
2054 {
2055 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2056 pVCpu->hm.s.fForceTLBFlush = false;
2057 }
2058 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2059 * where it is commented out. Support individual entry flushing
2060 * someday. */
2061#if 0
2062 else
2063 {
2064 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2065 {
2066 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2067 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2068 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2069 }
2070 else
2071 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2072
2073 pVCpu->hm.s.TlbShootdown.cPages = 0;
2074 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2075 }
2076#endif
2077}
2078
2079
2080/**
2081 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2082 *
2083 * @returns VBox status code.
2084 * @param pVM Pointer to the VM.
2085 * @param pVCpu Pointer to the VMCPU.
2086 * @param pCpu Pointer to the global HM CPU struct.
2087 *
2088 * @remarks Called with interrupts disabled.
2089 */
2090static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2091{
2092 AssertPtr(pVM);
2093 AssertPtr(pVCpu);
2094 AssertPtr(pCpu);
2095 Assert(pCpu->idCpu != NIL_RTCPUID);
2096 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2097 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2098
2099 /*
2100 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2101 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2102 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2103 */
2104 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2105 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2106 {
2107 pVCpu->hm.s.fForceTLBFlush = true;
2108 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2109 }
2110
2111 /* Check for explicit TLB shootdown flushes. */
2112 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2113 {
2114 /*
2115 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2116 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2117 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2118 */
2119 pVCpu->hm.s.fForceTLBFlush = true;
2120 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2121 }
2122
2123 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2124 if (pVCpu->hm.s.fForceTLBFlush)
2125 {
2126 ++pCpu->uCurrentAsid;
2127 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2128 {
2129 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2130 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2131 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2132 }
2133
2134 pVCpu->hm.s.fForceTLBFlush = false;
2135 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2136 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2137 if (pCpu->fFlushAsidBeforeUse)
2138 {
2139 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2140 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2141 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2142 {
2143 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2144 pCpu->fFlushAsidBeforeUse = false;
2145 }
2146 else
2147 {
2148 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2149 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2150 }
2151 }
2152 }
2153 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2154 * where it is commented out. Support individual entry flushing
2155 * someday. */
2156#if 0
2157 else
2158 {
2159 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2160 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2161 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2162 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2163
2164 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2165 {
2166 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2167 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2168 {
2169 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2170 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2171 }
2172 else
2173 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2174
2175 pVCpu->hm.s.TlbShootdown.cPages = 0;
2176 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2177 }
2178 else
2179 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2180 }
2181#endif
2182
2183 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2184 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2185 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2186 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2187 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2188 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2189 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2190
2191 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2192 AssertRC(rc);
2193}
2194
2195
2196/**
2197 * Flushes the guest TLB entry based on CPU capabilities.
2198 *
2199 * @param pVCpu Pointer to the VMCPU.
2200 * @param pCpu Pointer to the global HM CPU struct.
2201 */
2202DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2203{
2204#ifdef HMVMX_ALWAYS_FLUSH_TLB
2205 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2206#endif
2207 PVM pVM = pVCpu->CTX_SUFF(pVM);
2208 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2209 {
2210 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2211 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2212 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2213 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2214 default:
2215 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2216 break;
2217 }
2218
2219 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2220 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2221
2222 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2223}
2224
2225
2226/**
2227 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2228 * TLB entries from the host TLB before VM-entry.
2229 *
2230 * @returns VBox status code.
2231 * @param pVM Pointer to the VM.
2232 */
2233static int hmR0VmxSetupTaggedTlb(PVM pVM)
2234{
2235 /*
2236 * Determine optimal flush type for Nested Paging.
2237 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2238 * guest execution (see hmR3InitFinalizeR0()).
2239 */
2240 if (pVM->hm.s.fNestedPaging)
2241 {
2242 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2243 {
2244 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2245 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2246 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2247 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2248 else
2249 {
2250 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2251 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2252 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2253 }
2254
2255 /* Make sure the write-back cacheable memory type for EPT is supported. */
2256 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2257 {
2258 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2259 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2261 }
2262 }
2263 else
2264 {
2265 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2266 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2267 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2268 }
2269 }
2270
2271 /*
2272 * Determine optimal flush type for VPID.
2273 */
2274 if (pVM->hm.s.vmx.fVpid)
2275 {
2276 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2277 {
2278 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2279 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2280 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2281 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2282 else
2283 {
2284 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2285 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2286 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2287 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2288 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2289 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2290 pVM->hm.s.vmx.fVpid = false;
2291 }
2292 }
2293 else
2294 {
2295 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2296 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2297 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2298 pVM->hm.s.vmx.fVpid = false;
2299 }
2300 }
2301
2302 /*
2303 * Setup the handler for flushing tagged-TLBs.
2304 */
2305 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2306 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2307 else if (pVM->hm.s.fNestedPaging)
2308 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2309 else if (pVM->hm.s.vmx.fVpid)
2310 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2311 else
2312 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2313 return VINF_SUCCESS;
2314}
2315
2316
2317/**
2318 * Sets up pin-based VM-execution controls in the VMCS.
2319 *
2320 * @returns VBox status code.
2321 * @param pVM Pointer to the VM.
2322 * @param pVCpu Pointer to the VMCPU.
2323 */
2324static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2325{
2326 AssertPtr(pVM);
2327 AssertPtr(pVCpu);
2328
2329 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2330 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2331
2332 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2333 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2334
2335 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2336 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2337
2338 /* Enable the VMX preemption timer. */
2339 if (pVM->hm.s.vmx.fUsePreemptTimer)
2340 {
2341 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2342 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2343 }
2344
2345 if ((val & zap) != val)
2346 {
2347 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2348 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2349 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2350 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2351 }
2352
2353 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2354 AssertRCReturn(rc, rc);
2355
2356 pVCpu->hm.s.vmx.u32PinCtls = val;
2357 return rc;
2358}
2359
2360
2361/**
2362 * Sets up processor-based VM-execution controls in the VMCS.
2363 *
2364 * @returns VBox status code.
2365 * @param pVM Pointer to the VM.
2366 * @param pVMCPU Pointer to the VMCPU.
2367 */
2368static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2369{
2370 AssertPtr(pVM);
2371 AssertPtr(pVCpu);
2372
2373 int rc = VERR_INTERNAL_ERROR_5;
2374 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2375 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2376
2377 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2378 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2379 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2380 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2381 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2382 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2383 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2384
2385 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2386 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2387 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2388 {
2389 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2390 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2391 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2392 }
2393
2394 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2395 if (!pVM->hm.s.fNestedPaging)
2396 {
2397 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2398 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2399 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2400 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2401 }
2402
2403 /* Use TPR shadowing if supported by the CPU. */
2404 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2405 {
2406 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2407 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2408 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2409 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2410 AssertRCReturn(rc, rc);
2411
2412 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2413 /* CR8 writes cause a VM-exit based on TPR threshold. */
2414 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2415 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2416 }
2417 else
2418 {
2419 /*
2420 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2421 * Set this control only for 64-bit guests.
2422 */
2423 if (pVM->hm.s.fAllow64BitGuests)
2424 {
2425 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2426 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2427 }
2428 }
2429
2430 /* Use MSR-bitmaps if supported by the CPU. */
2431 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2432 {
2433 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2434
2435 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2436 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2437 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2438 AssertRCReturn(rc, rc);
2439
2440 /*
2441 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2442 * automatically using dedicated fields in the VMCS.
2443 */
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2449
2450#if HC_ARCH_BITS == 64
2451 /*
2452 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2453 */
2454 if (pVM->hm.s.fAllow64BitGuests)
2455 {
2456 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2457 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2458 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2459 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2460 }
2461#endif
2462 }
2463
2464 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2465 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2466 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2467
2468 if ((val & zap) != val)
2469 {
2470 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2471 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2472 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2473 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2474 }
2475
2476 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2477 AssertRCReturn(rc, rc);
2478
2479 pVCpu->hm.s.vmx.u32ProcCtls = val;
2480
2481 /*
2482 * Secondary processor-based VM-execution controls.
2483 */
2484 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2485 {
2486 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2487 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2488
2489 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2490 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2491
2492 if (pVM->hm.s.fNestedPaging)
2493 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2494 else
2495 {
2496 /*
2497 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2498 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2499 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2500 */
2501 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2502 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2503 }
2504
2505 if (pVM->hm.s.vmx.fVpid)
2506 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2507
2508 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2509 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2510
2511 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2512 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2513 * done dynamically. */
2514 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2515 {
2516 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2517 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2518 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2519 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2520 AssertRCReturn(rc, rc);
2521 }
2522
2523 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2524 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2525
2526 if ((val & zap) != val)
2527 {
2528 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2529 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2530 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2531 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2532 }
2533
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2535 AssertRCReturn(rc, rc);
2536
2537 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2538 }
2539 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2540 {
2541 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2542 "available\n"));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 return VINF_SUCCESS;
2548}
2549
2550
2551/**
2552 * Sets up miscellaneous (everything other than Pin & Processor-based
2553 * VM-execution) control fields in the VMCS.
2554 *
2555 * @returns VBox status code.
2556 * @param pVM Pointer to the VM.
2557 * @param pVCpu Pointer to the VMCPU.
2558 */
2559static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2560{
2561 NOREF(pVM);
2562 AssertPtr(pVM);
2563 AssertPtr(pVCpu);
2564
2565 int rc = VERR_GENERAL_FAILURE;
2566
2567 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2568#if 0
2569 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2570 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2571 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2572
2573 /*
2574 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2575 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2576 * We thus use the exception bitmap to control it rather than use both.
2577 */
2578 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2580
2581 /** @todo Explore possibility of using IO-bitmaps. */
2582 /* All IO & IOIO instructions cause VM-exits. */
2583 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2584 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2585
2586 /* Initialize the MSR-bitmap area. */
2587 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2588 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2590#endif
2591
2592 /* Setup MSR auto-load/store area. */
2593 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2594 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2595 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2596 AssertRCReturn(rc, rc);
2597 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2598 AssertRCReturn(rc, rc);
2599
2600 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2601 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2602 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2603 AssertRCReturn(rc, rc);
2604
2605 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2606 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2607 AssertRCReturn(rc, rc);
2608
2609 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2610#if 0
2611 /* Setup debug controls */
2612 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2613 AssertRCReturn(rc, rc);
2614 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2615 AssertRCReturn(rc, rc);
2616#endif
2617
2618 return rc;
2619}
2620
2621
2622/**
2623 * Sets up the initial exception bitmap in the VMCS based on static conditions
2624 * (i.e. conditions that cannot ever change after starting the VM).
2625 *
2626 * @returns VBox status code.
2627 * @param pVM Pointer to the VM.
2628 * @param pVCpu Pointer to the VMCPU.
2629 */
2630static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2631{
2632 AssertPtr(pVM);
2633 AssertPtr(pVCpu);
2634
2635 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2636
2637 uint32_t u32XcptBitmap = 0;
2638
2639 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2640 if (!pVM->hm.s.fNestedPaging)
2641 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2642
2643 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2644 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2645 AssertRCReturn(rc, rc);
2646 return rc;
2647}
2648
2649
2650/**
2651 * Sets up the initial guest-state mask. The guest-state mask is consulted
2652 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2653 * for the nested virtualization case (as it would cause a VM-exit).
2654 *
2655 * @param pVCpu Pointer to the VMCPU.
2656 */
2657static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2658{
2659 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2660 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2661 return VINF_SUCCESS;
2662}
2663
2664
2665/**
2666 * Does per-VM VT-x initialization.
2667 *
2668 * @returns VBox status code.
2669 * @param pVM Pointer to the VM.
2670 */
2671VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2672{
2673 LogFlowFunc(("pVM=%p\n", pVM));
2674
2675 int rc = hmR0VmxStructsAlloc(pVM);
2676 if (RT_FAILURE(rc))
2677 {
2678 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2679 return rc;
2680 }
2681
2682 return VINF_SUCCESS;
2683}
2684
2685
2686/**
2687 * Does per-VM VT-x termination.
2688 *
2689 * @returns VBox status code.
2690 * @param pVM Pointer to the VM.
2691 */
2692VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2693{
2694 LogFlowFunc(("pVM=%p\n", pVM));
2695
2696#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2697 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2698 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2699#endif
2700 hmR0VmxStructsFree(pVM);
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Sets up the VM for execution under VT-x.
2707 * This function is only called once per-VM during initialization.
2708 *
2709 * @returns VBox status code.
2710 * @param pVM Pointer to the VM.
2711 */
2712VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2713{
2714 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2715 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2716
2717 LogFlowFunc(("pVM=%p\n", pVM));
2718
2719 /*
2720 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2721 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2722 */
2723 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2724 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2725 || !pVM->hm.s.vmx.pRealModeTSS))
2726 {
2727 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2728 return VERR_INTERNAL_ERROR;
2729 }
2730
2731#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2732 /*
2733 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2734 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2735 */
2736 if ( pVM->hm.s.fAllow64BitGuests
2737 && !HMVMX_IS_64BIT_HOST_MODE())
2738 {
2739 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2740 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2741 }
2742#endif
2743
2744 /* Initialize these always, see hmR3InitFinalizeR0().*/
2745 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2746 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2747
2748 /* Setup the tagged-TLB flush handlers. */
2749 int rc = hmR0VmxSetupTaggedTlb(pVM);
2750 if (RT_FAILURE(rc))
2751 {
2752 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2753 return rc;
2754 }
2755
2756 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2757 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2758#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2759 if ( HMVMX_IS_64BIT_HOST_MODE()
2760 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2761 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2762 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2763 {
2764 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2765 }
2766#endif
2767
2768 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2769 {
2770 PVMCPU pVCpu = &pVM->aCpus[i];
2771 AssertPtr(pVCpu);
2772 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2773
2774 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2775 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2776
2777 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2778 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2779 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2780
2781 /* Set revision dword at the beginning of the VMCS structure. */
2782 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2783
2784 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2785 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2787 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2788
2789 /* Load this VMCS as the current VMCS. */
2790 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2791 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2792 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2793
2794 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2796 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2797
2798 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2800 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2801
2802 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2804 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2805
2806 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2815 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2816 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2817 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2818#endif
2819
2820 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2821 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2822 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2823 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2824
2825 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2826
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2828 }
2829
2830 return VINF_SUCCESS;
2831}
2832
2833
2834/**
2835 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2836 * the VMCS.
2837 *
2838 * @returns VBox status code.
2839 * @param pVM Pointer to the VM.
2840 * @param pVCpu Pointer to the VMCPU.
2841 */
2842DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2843{
2844 NOREF(pVM); NOREF(pVCpu);
2845
2846 RTCCUINTREG uReg = ASMGetCR0();
2847 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2848 AssertRCReturn(rc, rc);
2849
2850#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2851 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2852 if (HMVMX_IS_64BIT_HOST_MODE())
2853 {
2854 uint64_t uRegCR3 = HMR0Get64bitCR3();
2855 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2856 }
2857 else
2858#endif
2859 {
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 }
2863 AssertRCReturn(rc, rc);
2864
2865 uReg = ASMGetCR4();
2866 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2867 AssertRCReturn(rc, rc);
2868 return rc;
2869}
2870
2871
2872#if HC_ARCH_BITS == 64
2873/**
2874 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2875 * requirements. See hmR0VmxSaveHostSegmentRegs().
2876 */
2877# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2878 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2879 { \
2880 bool fValidSelector = true; \
2881 if ((selValue) & X86_SEL_LDT) \
2882 { \
2883 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2884 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2885 } \
2886 if (fValidSelector) \
2887 { \
2888 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2889 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2890 } \
2891 (selValue) = 0; \
2892 }
2893#endif
2894
2895
2896/**
2897 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2898 * the host-state area in the VMCS.
2899 *
2900 * @returns VBox status code.
2901 * @param pVM Pointer to the VM.
2902 * @param pVCpu Pointer to the VMCPU.
2903 */
2904DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2905{
2906 int rc = VERR_INTERNAL_ERROR_5;
2907
2908#if HC_ARCH_BITS == 64
2909 /*
2910 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2911 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2912 */
2913 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2914 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2915#endif
2916
2917 /*
2918 * Host DS, ES, FS and GS segment registers.
2919 */
2920#if HC_ARCH_BITS == 64
2921 RTSEL uSelDS = ASMGetDS();
2922 RTSEL uSelES = ASMGetES();
2923 RTSEL uSelFS = ASMGetFS();
2924 RTSEL uSelGS = ASMGetGS();
2925#else
2926 RTSEL uSelDS = 0;
2927 RTSEL uSelES = 0;
2928 RTSEL uSelFS = 0;
2929 RTSEL uSelGS = 0;
2930#endif
2931
2932 /* Recalculate which host-state bits need to be manually restored. */
2933 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2934
2935 /*
2936 * Host CS and SS segment registers.
2937 */
2938#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2939 RTSEL uSelCS;
2940 RTSEL uSelSS;
2941 if (HMVMX_IS_64BIT_HOST_MODE())
2942 {
2943 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2944 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2945 }
2946 else
2947 {
2948 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2949 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2950 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2951 }
2952#else
2953 RTSEL uSelCS = ASMGetCS();
2954 RTSEL uSelSS = ASMGetSS();
2955#endif
2956
2957 /*
2958 * Host TR segment register.
2959 */
2960 RTSEL uSelTR = ASMGetTR();
2961
2962#if HC_ARCH_BITS == 64
2963 /*
2964 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2965 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2966 */
2967 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2968 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2969 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2970 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2971# undef VMXLOCAL_ADJUST_HOST_SEG
2972#endif
2973
2974 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2975 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2976 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2977 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2978 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2979 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2980 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2981 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2982 Assert(uSelCS);
2983 Assert(uSelTR);
2984
2985 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2986#if 0
2987 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2988 Assert(uSelSS != 0);
2989#endif
2990
2991 /* Write these host selector fields into the host-state area in the VMCS. */
2992 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2994#if HC_ARCH_BITS == 64
2995 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2999#endif
3000 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3001
3002 /*
3003 * Host GDTR and IDTR.
3004 */
3005 RTGDTR Gdtr;
3006 RT_ZERO(Gdtr);
3007#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3008 if (HMVMX_IS_64BIT_HOST_MODE())
3009 {
3010 X86XDTR64 Gdtr64;
3011 X86XDTR64 Idtr64;
3012 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3013 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3014 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3015
3016 Gdtr.cbGdt = Gdtr64.cb;
3017 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3018 }
3019 else
3020#endif
3021 {
3022 RTIDTR Idtr;
3023 ASMGetGDTR(&Gdtr);
3024 ASMGetIDTR(&Idtr);
3025 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3026 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3027
3028#if HC_ARCH_BITS == 64
3029 /*
3030 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3031 * maximum limit (0xffff) on every VM-exit.
3032 */
3033 if (Gdtr.cbGdt != 0xffff)
3034 {
3035 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3036 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3037 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3038 }
3039
3040 /*
3041 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3042 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3043 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3044 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3045 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3046 * hosts where we are pretty sure it won't cause trouble.
3047 */
3048# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3049 if (Idtr.cbIdt < 0x0fff)
3050# else
3051 if (Idtr.cbIdt != 0xffff)
3052# endif
3053 {
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3055 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3056 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3057 }
3058#endif
3059 }
3060
3061 /*
3062 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3063 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3064 */
3065 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3066 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3067 VERR_VMX_INVALID_HOST_STATE);
3068
3069 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3070#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3071 if (HMVMX_IS_64BIT_HOST_MODE())
3072 {
3073 /* We need the 64-bit TR base for hybrid darwin. */
3074 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3075 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3076 }
3077 else
3078#endif
3079 {
3080 uintptr_t uTRBase;
3081#if HC_ARCH_BITS == 64
3082 uTRBase = X86DESC64_BASE(pDesc);
3083
3084 /*
3085 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3086 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3087 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3088 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3089 *
3090 * [1] See Intel spec. 3.5 "System Descriptor Types".
3091 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3092 */
3093 Assert(pDesc->System.u4Type == 11);
3094 if ( pDesc->System.u16LimitLow != 0x67
3095 || pDesc->System.u4LimitHigh)
3096 {
3097 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3098 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3099 if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3100 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3101 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3102
3103 /* Store the GDTR here as we need it while restoring TR. */
3104 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3105 }
3106#else
3107 uTRBase = X86DESC_BASE(pDesc);
3108#endif
3109 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3110 }
3111 AssertRCReturn(rc, rc);
3112
3113 /*
3114 * Host FS base and GS base.
3115 */
3116#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3117 if (HMVMX_IS_64BIT_HOST_MODE())
3118 {
3119 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3120 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3121 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3122 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3123
3124# if HC_ARCH_BITS == 64
3125 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3126 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3127 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3128 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3129 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3130# endif
3131 }
3132#endif
3133 return rc;
3134}
3135
3136
3137/**
3138 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3139 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3140 * the host after every successful VM-exit.
3141 *
3142 * @returns VBox status code.
3143 * @param pVM Pointer to the VM.
3144 * @param pVCpu Pointer to the VMCPU.
3145 *
3146 * @remarks No-long-jump zone!!!
3147 */
3148DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3149{
3150 NOREF(pVM);
3151
3152 AssertPtr(pVCpu);
3153 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3154
3155 int rc = VINF_SUCCESS;
3156#if HC_ARCH_BITS == 64
3157 if (pVM->hm.s.fAllow64BitGuests)
3158 hmR0VmxLazySaveHostMsrs(pVCpu);
3159#endif
3160
3161 /*
3162 * Host Sysenter MSRs.
3163 */
3164 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3165 AssertRCReturn(rc, rc);
3166#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3167 if (HMVMX_IS_64BIT_HOST_MODE())
3168 {
3169 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3170 AssertRCReturn(rc, rc);
3171 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3172 }
3173 else
3174 {
3175 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3176 AssertRCReturn(rc, rc);
3177 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3178 }
3179#elif HC_ARCH_BITS == 32
3180 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3181 AssertRCReturn(rc, rc);
3182 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3183#else
3184 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3185 AssertRCReturn(rc, rc);
3186 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3187#endif
3188 AssertRCReturn(rc, rc);
3189
3190 /*
3191 * Host EFER MSR.
3192 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3193 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3194 */
3195 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3196 {
3197 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3198 AssertRCReturn(rc, rc);
3199 }
3200
3201 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3202 * hmR0VmxLoadGuestExitCtls() !! */
3203
3204 return rc;
3205}
3206
3207
3208/**
3209 * Figures out if we need to swap the EFER MSR which is
3210 * particularly expensive.
3211 *
3212 * We check all relevant bits. For now, that's everything
3213 * besides LMA/LME, as these two bits are handled by VM-entry,
3214 * see hmR0VmxLoadGuestExitCtls() and
3215 * hmR0VMxLoadGuestEntryCtls().
3216 *
3217 * @returns true if we need to load guest EFER, false otherwise.
3218 * @param pVCpu Pointer to the VMCPU.
3219 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3220 * out-of-sync. Make sure to update the required fields
3221 * before using them.
3222 *
3223 * @remarks Requires EFER, CR4.
3224 * @remarks No-long-jump zone!!!
3225 */
3226static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3227{
3228#ifdef HMVMX_ALWAYS_SWAP_EFER
3229 return true;
3230#endif
3231
3232#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3233 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3234 if (CPUMIsGuestInLongMode(pVCpu))
3235 return false;
3236#endif
3237
3238 PVM pVM = pVCpu->CTX_SUFF(pVM);
3239 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3240 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3241
3242 /*
3243 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3244 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3245 */
3246 if ( CPUMIsGuestInLongMode(pVCpu)
3247 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3248 {
3249 return true;
3250 }
3251
3252 /*
3253 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3254 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3255 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3256 */
3257 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3258 && (pMixedCtx->cr0 & X86_CR0_PG)
3259 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3260 {
3261 /* Assert that host is PAE capable. */
3262 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3263 return true;
3264 }
3265
3266 /** @todo Check the latest Intel spec. for any other bits,
3267 * like SMEP/SMAP? */
3268 return false;
3269}
3270
3271
3272/**
3273 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3274 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3275 * controls".
3276 *
3277 * @returns VBox status code.
3278 * @param pVCpu Pointer to the VMCPU.
3279 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3280 * out-of-sync. Make sure to update the required fields
3281 * before using them.
3282 *
3283 * @remarks Requires EFER.
3284 * @remarks No-long-jump zone!!!
3285 */
3286DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3287{
3288 int rc = VINF_SUCCESS;
3289 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3290 {
3291 PVM pVM = pVCpu->CTX_SUFF(pVM);
3292 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3293 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3294
3295 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3296 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3297
3298 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3299 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3300 {
3301 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3302 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3303 }
3304 else
3305 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3306
3307 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3308 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3309 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3310 {
3311 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3312 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3313 }
3314
3315 /*
3316 * The following should -not- be set (since we're not in SMM mode):
3317 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3318 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3319 */
3320
3321 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3322 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3323
3324 if ((val & zap) != val)
3325 {
3326 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3327 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3328 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3329 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3330 }
3331
3332 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3333 AssertRCReturn(rc, rc);
3334
3335 pVCpu->hm.s.vmx.u32EntryCtls = val;
3336 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3337 }
3338 return rc;
3339}
3340
3341
3342/**
3343 * Sets up the VM-exit controls in the VMCS.
3344 *
3345 * @returns VBox status code.
3346 * @param pVM Pointer to the VM.
3347 * @param pVCpu Pointer to the VMCPU.
3348 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3349 * out-of-sync. Make sure to update the required fields
3350 * before using them.
3351 *
3352 * @remarks Requires EFER.
3353 */
3354DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3355{
3356 NOREF(pMixedCtx);
3357
3358 int rc = VINF_SUCCESS;
3359 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3360 {
3361 PVM pVM = pVCpu->CTX_SUFF(pVM);
3362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3364
3365 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3366 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3367
3368 /*
3369 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3370 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3371 */
3372#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3373 if (HMVMX_IS_64BIT_HOST_MODE())
3374 {
3375 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3376 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3377 }
3378 else
3379 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3380#else
3381 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3382 {
3383 /* The switcher returns to long mode, EFER is managed by the switcher. */
3384 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3385 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3386 }
3387 else
3388 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3389#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3390
3391 /* If the newer VMCS fields for managing EFER exists, use it. */
3392 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3393 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3394 {
3395 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3396 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3397 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3398 }
3399
3400 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3401 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3402
3403 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3404 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3405 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3406
3407 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3408 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3409
3410 if ((val & zap) != val)
3411 {
3412 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3413 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3414 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3415 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3416 }
3417
3418 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3419 AssertRCReturn(rc, rc);
3420
3421 pVCpu->hm.s.vmx.u32ExitCtls = val;
3422 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3423 }
3424 return rc;
3425}
3426
3427
3428/**
3429 * Loads the guest APIC and related state.
3430 *
3431 * @returns VBox status code.
3432 * @param pVM Pointer to the VM.
3433 * @param pVCpu Pointer to the VMCPU.
3434 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3435 * out-of-sync. Make sure to update the required fields
3436 * before using them.
3437 */
3438DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3439{
3440 NOREF(pMixedCtx);
3441
3442 int rc = VINF_SUCCESS;
3443 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3444 {
3445 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3446 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3447 {
3448 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3449
3450 bool fPendingIntr = false;
3451 uint8_t u8Tpr = 0;
3452 uint8_t u8PendingIntr = 0;
3453 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3454 AssertRCReturn(rc, rc);
3455
3456 /*
3457 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3458 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3459 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3460 * the interrupt when we VM-exit for other reasons.
3461 */
3462 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3463 uint32_t u32TprThreshold = 0;
3464 if (fPendingIntr)
3465 {
3466 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3467 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3468 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3469 if (u8PendingPriority <= u8TprPriority)
3470 u32TprThreshold = u8PendingPriority;
3471 else
3472 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3473 }
3474 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3475
3476 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3477 AssertRCReturn(rc, rc);
3478 }
3479
3480 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3481 }
3482 return rc;
3483}
3484
3485
3486/**
3487 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3488 *
3489 * @returns Guest's interruptibility-state.
3490 * @param pVCpu Pointer to the VMCPU.
3491 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3492 * out-of-sync. Make sure to update the required fields
3493 * before using them.
3494 *
3495 * @remarks No-long-jump zone!!!
3496 */
3497DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3498{
3499 /*
3500 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3501 */
3502 uint32_t uIntrState = 0;
3503 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3504 {
3505 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3506 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3507 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3508 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3509 {
3510 if (pMixedCtx->eflags.Bits.u1IF)
3511 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3512 else
3513 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3514 }
3515 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3516 }
3517
3518 /*
3519 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3520 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3521 * setting this would block host-NMIs and IRET will not clear the blocking.
3522 *
3523 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3524 */
3525 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3526 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3527 {
3528 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3529 }
3530
3531 return uIntrState;
3532}
3533
3534
3535/**
3536 * Loads the guest's interruptibility-state into the guest-state area in the
3537 * VMCS.
3538 *
3539 * @returns VBox status code.
3540 * @param pVCpu Pointer to the VMCPU.
3541 * @param uIntrState The interruptibility-state to set.
3542 */
3543static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3544{
3545 NOREF(pVCpu);
3546 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3547 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3548 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3549 AssertRCReturn(rc, rc);
3550 return rc;
3551}
3552
3553
3554/**
3555 * Loads the guest's RIP into the guest-state area in the VMCS.
3556 *
3557 * @returns VBox status code.
3558 * @param pVCpu Pointer to the VMCPU.
3559 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3560 * out-of-sync. Make sure to update the required fields
3561 * before using them.
3562 *
3563 * @remarks No-long-jump zone!!!
3564 */
3565static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3566{
3567 int rc = VINF_SUCCESS;
3568 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3569 {
3570 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3571 AssertRCReturn(rc, rc);
3572
3573 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3574 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3575 HMCPU_CF_VALUE(pVCpu)));
3576 }
3577 return rc;
3578}
3579
3580
3581/**
3582 * Loads the guest's RSP into the guest-state area in the VMCS.
3583 *
3584 * @returns VBox status code.
3585 * @param pVCpu Pointer to the VMCPU.
3586 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3587 * out-of-sync. Make sure to update the required fields
3588 * before using them.
3589 *
3590 * @remarks No-long-jump zone!!!
3591 */
3592static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3593{
3594 int rc = VINF_SUCCESS;
3595 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3596 {
3597 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3598 AssertRCReturn(rc, rc);
3599
3600 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3601 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3602 }
3603 return rc;
3604}
3605
3606
3607/**
3608 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3609 *
3610 * @returns VBox status code.
3611 * @param pVCpu Pointer to the VMCPU.
3612 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3613 * out-of-sync. Make sure to update the required fields
3614 * before using them.
3615 *
3616 * @remarks No-long-jump zone!!!
3617 */
3618static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3619{
3620 int rc = VINF_SUCCESS;
3621 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3622 {
3623 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3624 Let us assert it as such and use 32-bit VMWRITE. */
3625 Assert(!(pMixedCtx->rflags.u64 >> 32));
3626 X86EFLAGS Eflags = pMixedCtx->eflags;
3627 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3628 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3629 * These will never be cleared/set, unless some other part of the VMM
3630 * code is buggy - in which case we're better of finding and fixing
3631 * those bugs than hiding them. */
3632 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3633 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3634 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3635 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3636
3637 /*
3638 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3639 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3640 */
3641 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3642 {
3643 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3644 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3645 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3646 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3647 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3648 }
3649
3650 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3651 AssertRCReturn(rc, rc);
3652
3653 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3654 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3655 }
3656 return rc;
3657}
3658
3659
3660/**
3661 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3662 *
3663 * @returns VBox status code.
3664 * @param pVCpu Pointer to the VMCPU.
3665 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3666 * out-of-sync. Make sure to update the required fields
3667 * before using them.
3668 *
3669 * @remarks No-long-jump zone!!!
3670 */
3671DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3672{
3673 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3674 AssertRCReturn(rc, rc);
3675 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3676 AssertRCReturn(rc, rc);
3677 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3678 AssertRCReturn(rc, rc);
3679 return rc;
3680}
3681
3682
3683/**
3684 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3685 * CR0 is partially shared with the host and we have to consider the FPU bits.
3686 *
3687 * @returns VBox status code.
3688 * @param pVM Pointer to the VM.
3689 * @param pVCpu Pointer to the VMCPU.
3690 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3691 * out-of-sync. Make sure to update the required fields
3692 * before using them.
3693 *
3694 * @remarks No-long-jump zone!!!
3695 */
3696static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3697{
3698 /*
3699 * Guest CR0.
3700 * Guest FPU.
3701 */
3702 int rc = VINF_SUCCESS;
3703 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3704 {
3705 Assert(!(pMixedCtx->cr0 >> 32));
3706 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3707 PVM pVM = pVCpu->CTX_SUFF(pVM);
3708
3709 /* The guest's view (read access) of its CR0 is unblemished. */
3710 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3711 AssertRCReturn(rc, rc);
3712 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3713
3714 /* Setup VT-x's view of the guest CR0. */
3715 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3716 if (pVM->hm.s.fNestedPaging)
3717 {
3718 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3719 {
3720 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3721 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3723 }
3724 else
3725 {
3726 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3727 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3728 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729 }
3730
3731 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3732 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3733 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734
3735 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3736 AssertRCReturn(rc, rc);
3737 }
3738 else
3739 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3740
3741 /*
3742 * Guest FPU bits.
3743 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3744 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3745 */
3746 u32GuestCR0 |= X86_CR0_NE;
3747 bool fInterceptNM = false;
3748 if (CPUMIsGuestFPUStateActive(pVCpu))
3749 {
3750 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3751 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3752 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3753 }
3754 else
3755 {
3756 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3757 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3758 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3759 }
3760
3761 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3762 bool fInterceptMF = false;
3763 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3764 fInterceptMF = true;
3765
3766 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3767 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3768 {
3769 Assert(PDMVmmDevHeapIsEnabled(pVM));
3770 Assert(pVM->hm.s.vmx.pRealModeTSS);
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3772 fInterceptNM = true;
3773 fInterceptMF = true;
3774 }
3775 else
3776 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3777
3778 if (fInterceptNM)
3779 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3780 else
3781 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3782
3783 if (fInterceptMF)
3784 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3785 else
3786 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3787
3788 /* Additional intercepts for debugging, define these yourself explicitly. */
3789#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3790 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3791 | RT_BIT(X86_XCPT_BP)
3792 | RT_BIT(X86_XCPT_DB)
3793 | RT_BIT(X86_XCPT_DE)
3794 | RT_BIT(X86_XCPT_NM)
3795 | RT_BIT(X86_XCPT_TS)
3796 | RT_BIT(X86_XCPT_UD)
3797 | RT_BIT(X86_XCPT_NP)
3798 | RT_BIT(X86_XCPT_SS)
3799 | RT_BIT(X86_XCPT_GP)
3800 | RT_BIT(X86_XCPT_PF)
3801 | RT_BIT(X86_XCPT_MF)
3802 ;
3803#elif defined(HMVMX_ALWAYS_TRAP_PF)
3804 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3805#endif
3806
3807 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3808
3809 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3810 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3811 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3812 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3813 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3814 else
3815 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3816
3817 u32GuestCR0 |= uSetCR0;
3818 u32GuestCR0 &= uZapCR0;
3819 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3820
3821 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3822 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3823 AssertRCReturn(rc, rc);
3824 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3825 AssertRCReturn(rc, rc);
3826 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3827 uZapCR0));
3828
3829 /*
3830 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3831 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3832 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3833 */
3834 uint32_t u32CR0Mask = 0;
3835 u32CR0Mask = X86_CR0_PE
3836 | X86_CR0_NE
3837 | X86_CR0_WP
3838 | X86_CR0_PG
3839 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3840 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3841 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3842
3843 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3844 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3845 * and @bugref{6944}. */
3846#if 0
3847 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3848 u32CR0Mask &= ~X86_CR0_PE;
3849#endif
3850 if (pVM->hm.s.fNestedPaging)
3851 u32CR0Mask &= ~X86_CR0_WP;
3852
3853 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3854 if (fInterceptNM)
3855 {
3856 u32CR0Mask |= X86_CR0_TS
3857 | X86_CR0_MP;
3858 }
3859
3860 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3861 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3862 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3863 AssertRCReturn(rc, rc);
3864 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3865
3866 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3867 }
3868 return rc;
3869}
3870
3871
3872/**
3873 * Loads the guest control registers (CR3, CR4) into the guest-state area
3874 * in the VMCS.
3875 *
3876 * @returns VBox status code.
3877 * @param pVM Pointer to the VM.
3878 * @param pVCpu Pointer to the VMCPU.
3879 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3880 * out-of-sync. Make sure to update the required fields
3881 * before using them.
3882 *
3883 * @remarks No-long-jump zone!!!
3884 */
3885static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3886{
3887 int rc = VINF_SUCCESS;
3888 PVM pVM = pVCpu->CTX_SUFF(pVM);
3889
3890 /*
3891 * Guest CR2.
3892 * It's always loaded in the assembler code. Nothing to do here.
3893 */
3894
3895 /*
3896 * Guest CR3.
3897 */
3898 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3899 {
3900 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3901 if (pVM->hm.s.fNestedPaging)
3902 {
3903 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3904
3905 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3906 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3907 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3908 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3909
3910 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3911 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3912 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3913
3914 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3915 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3916 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3917 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3918
3919 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3920 AssertRCReturn(rc, rc);
3921 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3922
3923 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3924 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3925 {
3926 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3927 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3928 {
3929 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3930 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3931 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3932 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3933 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3934 }
3935
3936 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3937 have Unrestricted Execution to handle the guest when it's not using paging. */
3938 GCPhysGuestCR3 = pMixedCtx->cr3;
3939 }
3940 else
3941 {
3942 /*
3943 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3944 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3945 * EPT takes care of translating it to host-physical addresses.
3946 */
3947 RTGCPHYS GCPhys;
3948 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3949 Assert(PDMVmmDevHeapIsEnabled(pVM));
3950
3951 /* We obtain it here every time as the guest could have relocated this PCI region. */
3952 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3953 AssertRCReturn(rc, rc);
3954
3955 GCPhysGuestCR3 = GCPhys;
3956 }
3957
3958 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3959 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3960 }
3961 else
3962 {
3963 /* Non-nested paging case, just use the hypervisor's CR3. */
3964 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3965
3966 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3967 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3968 }
3969 AssertRCReturn(rc, rc);
3970
3971 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3972 }
3973
3974 /*
3975 * Guest CR4.
3976 */
3977 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3978 {
3979 Assert(!(pMixedCtx->cr4 >> 32));
3980 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3981
3982 /* The guest's view of its CR4 is unblemished. */
3983 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3984 AssertRCReturn(rc, rc);
3985 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3986
3987 /* Setup VT-x's view of the guest CR4. */
3988 /*
3989 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3990 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3991 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3992 */
3993 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3994 {
3995 Assert(pVM->hm.s.vmx.pRealModeTSS);
3996 Assert(PDMVmmDevHeapIsEnabled(pVM));
3997 u32GuestCR4 &= ~X86_CR4_VME;
3998 }
3999
4000 if (pVM->hm.s.fNestedPaging)
4001 {
4002 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4003 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4004 {
4005 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4006 u32GuestCR4 |= X86_CR4_PSE;
4007 /* Our identity mapping is a 32-bit page directory. */
4008 u32GuestCR4 &= ~X86_CR4_PAE;
4009 }
4010 /* else use guest CR4.*/
4011 }
4012 else
4013 {
4014 /*
4015 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4016 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4017 */
4018 switch (pVCpu->hm.s.enmShadowMode)
4019 {
4020 case PGMMODE_REAL: /* Real-mode. */
4021 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4022 case PGMMODE_32_BIT: /* 32-bit paging. */
4023 {
4024 u32GuestCR4 &= ~X86_CR4_PAE;
4025 break;
4026 }
4027
4028 case PGMMODE_PAE: /* PAE paging. */
4029 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4030 {
4031 u32GuestCR4 |= X86_CR4_PAE;
4032 break;
4033 }
4034
4035 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4036 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4037#ifdef VBOX_ENABLE_64_BITS_GUESTS
4038 break;
4039#endif
4040 default:
4041 AssertFailed();
4042 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4043 }
4044 }
4045
4046 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4047 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4048 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4049 u32GuestCR4 |= uSetCR4;
4050 u32GuestCR4 &= uZapCR4;
4051
4052 /* Write VT-x's view of the guest CR4 into the VMCS. */
4053 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4054 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4055 AssertRCReturn(rc, rc);
4056
4057 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4058 uint32_t u32CR4Mask = 0;
4059 u32CR4Mask = X86_CR4_VME
4060 | X86_CR4_PAE
4061 | X86_CR4_PGE
4062 | X86_CR4_PSE
4063 | X86_CR4_VMXE;
4064 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4065 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4066 AssertRCReturn(rc, rc);
4067
4068 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4069 }
4070 return rc;
4071}
4072
4073
4074/**
4075 * Loads the guest debug registers into the guest-state area in the VMCS.
4076 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4077 *
4078 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4079 *
4080 * @returns VBox status code.
4081 * @param pVCpu Pointer to the VMCPU.
4082 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4083 * out-of-sync. Make sure to update the required fields
4084 * before using them.
4085 *
4086 * @remarks No-long-jump zone!!!
4087 */
4088static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4089{
4090 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4091 return VINF_SUCCESS;
4092
4093#ifdef VBOX_STRICT
4094 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4095 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4096 {
4097 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4098 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4099 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4100 }
4101#endif
4102
4103 int rc;
4104 PVM pVM = pVCpu->CTX_SUFF(pVM);
4105 bool fInterceptDB = false;
4106 bool fInterceptMovDRx = false;
4107 if ( pVCpu->hm.s.fSingleInstruction
4108 || DBGFIsStepping(pVCpu))
4109 {
4110 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4111 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4112 {
4113 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4114 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4115 AssertRCReturn(rc, rc);
4116 Assert(fInterceptDB == false);
4117 }
4118 else
4119 {
4120 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4121 pVCpu->hm.s.fClearTrapFlag = true;
4122 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4123 fInterceptDB = true;
4124 }
4125 }
4126
4127 if ( fInterceptDB
4128 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4129 {
4130 /*
4131 * Use the combined guest and host DRx values found in the hypervisor
4132 * register set because the debugger has breakpoints active or someone
4133 * is single stepping on the host side without a monitor trap flag.
4134 *
4135 * Note! DBGF expects a clean DR6 state before executing guest code.
4136 */
4137#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4138 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4139 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4140 {
4141 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4142 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4143 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4144 }
4145 else
4146#endif
4147 if (!CPUMIsHyperDebugStateActive(pVCpu))
4148 {
4149 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4150 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4151 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4152 }
4153
4154 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4155 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4156 AssertRCReturn(rc, rc);
4157
4158 pVCpu->hm.s.fUsingHyperDR7 = true;
4159 fInterceptDB = true;
4160 fInterceptMovDRx = true;
4161 }
4162 else
4163 {
4164 /*
4165 * If the guest has enabled debug registers, we need to load them prior to
4166 * executing guest code so they'll trigger at the right time.
4167 */
4168 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4169 {
4170#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4171 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4172 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4173 {
4174 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4175 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4176 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4177 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4178 }
4179 else
4180#endif
4181 if (!CPUMIsGuestDebugStateActive(pVCpu))
4182 {
4183 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4184 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4185 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4186 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4187 }
4188 Assert(!fInterceptDB);
4189 Assert(!fInterceptMovDRx);
4190 }
4191 /*
4192 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4193 * must intercept #DB in order to maintain a correct DR6 guest value.
4194 */
4195#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4196 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4197 && !CPUMIsGuestDebugStateActive(pVCpu))
4198#else
4199 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4200#endif
4201 {
4202 fInterceptMovDRx = true;
4203 fInterceptDB = true;
4204 }
4205
4206 /* Update guest DR7. */
4207 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4208 AssertRCReturn(rc, rc);
4209
4210 pVCpu->hm.s.fUsingHyperDR7 = false;
4211 }
4212
4213 /*
4214 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4215 */
4216 if (fInterceptDB)
4217 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4218 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4219 {
4220#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4221 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4222#endif
4223 }
4224 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4225 AssertRCReturn(rc, rc);
4226
4227 /*
4228 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4229 */
4230 if (fInterceptMovDRx)
4231 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4232 else
4233 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4234 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4235 AssertRCReturn(rc, rc);
4236
4237 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4238 return VINF_SUCCESS;
4239}
4240
4241
4242#ifdef VBOX_STRICT
4243/**
4244 * Strict function to validate segment registers.
4245 *
4246 * @remarks ASSUMES CR0 is up to date.
4247 */
4248static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4249{
4250 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4251 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4252 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4253 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4254 && ( !CPUMIsGuestInRealModeEx(pCtx)
4255 && !CPUMIsGuestInV86ModeEx(pCtx)))
4256 {
4257 /* Protected mode checks */
4258 /* CS */
4259 Assert(pCtx->cs.Attr.n.u1Present);
4260 Assert(!(pCtx->cs.Attr.u & 0xf00));
4261 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4262 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4263 || !(pCtx->cs.Attr.n.u1Granularity));
4264 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4265 || (pCtx->cs.Attr.n.u1Granularity));
4266 /* CS cannot be loaded with NULL in protected mode. */
4267 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4268 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4269 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4270 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4271 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4272 else
4273 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4274 /* SS */
4275 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4276 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4277 if ( !(pCtx->cr0 & X86_CR0_PE)
4278 || pCtx->cs.Attr.n.u4Type == 3)
4279 {
4280 Assert(!pCtx->ss.Attr.n.u2Dpl);
4281 }
4282 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4283 {
4284 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4285 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4286 Assert(pCtx->ss.Attr.n.u1Present);
4287 Assert(!(pCtx->ss.Attr.u & 0xf00));
4288 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4289 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4290 || !(pCtx->ss.Attr.n.u1Granularity));
4291 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4292 || (pCtx->ss.Attr.n.u1Granularity));
4293 }
4294 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4295 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4296 {
4297 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4298 Assert(pCtx->ds.Attr.n.u1Present);
4299 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4300 Assert(!(pCtx->ds.Attr.u & 0xf00));
4301 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4302 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4303 || !(pCtx->ds.Attr.n.u1Granularity));
4304 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4305 || (pCtx->ds.Attr.n.u1Granularity));
4306 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4307 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4308 }
4309 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4310 {
4311 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4312 Assert(pCtx->es.Attr.n.u1Present);
4313 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4314 Assert(!(pCtx->es.Attr.u & 0xf00));
4315 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4316 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4317 || !(pCtx->es.Attr.n.u1Granularity));
4318 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4319 || (pCtx->es.Attr.n.u1Granularity));
4320 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4321 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4322 }
4323 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4324 {
4325 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4326 Assert(pCtx->fs.Attr.n.u1Present);
4327 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4328 Assert(!(pCtx->fs.Attr.u & 0xf00));
4329 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4330 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4331 || !(pCtx->fs.Attr.n.u1Granularity));
4332 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4333 || (pCtx->fs.Attr.n.u1Granularity));
4334 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4335 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4336 }
4337 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4338 {
4339 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4340 Assert(pCtx->gs.Attr.n.u1Present);
4341 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4342 Assert(!(pCtx->gs.Attr.u & 0xf00));
4343 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4344 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4345 || !(pCtx->gs.Attr.n.u1Granularity));
4346 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4347 || (pCtx->gs.Attr.n.u1Granularity));
4348 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4349 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4350 }
4351 /* 64-bit capable CPUs. */
4352# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4353 if (HMVMX_IS_64BIT_HOST_MODE())
4354 {
4355 Assert(!(pCtx->cs.u64Base >> 32));
4356 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4357 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4358 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4359 }
4360# endif
4361 }
4362 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4363 || ( CPUMIsGuestInRealModeEx(pCtx)
4364 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4365 {
4366 /* Real and v86 mode checks. */
4367 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4368 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4369 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4370 {
4371 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4372 }
4373 else
4374 {
4375 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4376 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4377 }
4378
4379 /* CS */
4380 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4381 Assert(pCtx->cs.u32Limit == 0xffff);
4382 Assert(u32CSAttr == 0xf3);
4383 /* SS */
4384 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4385 Assert(pCtx->ss.u32Limit == 0xffff);
4386 Assert(u32SSAttr == 0xf3);
4387 /* DS */
4388 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4389 Assert(pCtx->ds.u32Limit == 0xffff);
4390 Assert(u32DSAttr == 0xf3);
4391 /* ES */
4392 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4393 Assert(pCtx->es.u32Limit == 0xffff);
4394 Assert(u32ESAttr == 0xf3);
4395 /* FS */
4396 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4397 Assert(pCtx->fs.u32Limit == 0xffff);
4398 Assert(u32FSAttr == 0xf3);
4399 /* GS */
4400 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4401 Assert(pCtx->gs.u32Limit == 0xffff);
4402 Assert(u32GSAttr == 0xf3);
4403 /* 64-bit capable CPUs. */
4404# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4405 if (HMVMX_IS_64BIT_HOST_MODE())
4406 {
4407 Assert(!(pCtx->cs.u64Base >> 32));
4408 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4409 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4410 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4411 }
4412# endif
4413 }
4414}
4415#endif /* VBOX_STRICT */
4416
4417
4418/**
4419 * Writes a guest segment register into the guest-state area in the VMCS.
4420 *
4421 * @returns VBox status code.
4422 * @param pVCpu Pointer to the VMCPU.
4423 * @param idxSel Index of the selector in the VMCS.
4424 * @param idxLimit Index of the segment limit in the VMCS.
4425 * @param idxBase Index of the segment base in the VMCS.
4426 * @param idxAccess Index of the access rights of the segment in the VMCS.
4427 * @param pSelReg Pointer to the segment selector.
4428 *
4429 * @remarks No-long-jump zone!!!
4430 */
4431static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4432 uint32_t idxAccess, PCPUMSELREG pSelReg)
4433{
4434 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4435 AssertRCReturn(rc, rc);
4436 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4437 AssertRCReturn(rc, rc);
4438 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4439 AssertRCReturn(rc, rc);
4440
4441 uint32_t u32Access = pSelReg->Attr.u;
4442 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4443 {
4444 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4445 u32Access = 0xf3;
4446 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4447 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4448 }
4449 else
4450 {
4451 /*
4452 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4453 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4454 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4455 * loaded in protected-mode have their attribute as 0.
4456 */
4457 if (!u32Access)
4458 u32Access = X86DESCATTR_UNUSABLE;
4459 }
4460
4461 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4462 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4463 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4464
4465 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4466 AssertRCReturn(rc, rc);
4467 return rc;
4468}
4469
4470
4471/**
4472 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4473 * into the guest-state area in the VMCS.
4474 *
4475 * @returns VBox status code.
4476 * @param pVM Pointer to the VM.
4477 * @param pVCPU Pointer to the VMCPU.
4478 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4479 * out-of-sync. Make sure to update the required fields
4480 * before using them.
4481 *
4482 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4483 * @remarks No-long-jump zone!!!
4484 */
4485static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4486{
4487 int rc = VERR_INTERNAL_ERROR_5;
4488 PVM pVM = pVCpu->CTX_SUFF(pVM);
4489
4490 /*
4491 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4492 */
4493 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4494 {
4495 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4496 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4497 {
4498 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4501 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4502 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4503 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4504 }
4505
4506#ifdef VBOX_WITH_REM
4507 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4508 {
4509 Assert(pVM->hm.s.vmx.pRealModeTSS);
4510 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4511 if ( pVCpu->hm.s.vmx.fWasInRealMode
4512 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4513 {
4514 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4515 in real-mode (e.g. OpenBSD 4.0) */
4516 REMFlushTBs(pVM);
4517 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4518 pVCpu->hm.s.vmx.fWasInRealMode = false;
4519 }
4520 }
4521#endif
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4523 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4526 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4529 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4530 AssertRCReturn(rc, rc);
4531 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4532 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4533 AssertRCReturn(rc, rc);
4534 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4535 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4536 AssertRCReturn(rc, rc);
4537 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4538 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4539 AssertRCReturn(rc, rc);
4540
4541#ifdef VBOX_STRICT
4542 /* Validate. */
4543 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4544#endif
4545
4546 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4547 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4548 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4549 }
4550
4551 /*
4552 * Guest TR.
4553 */
4554 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4555 {
4556 /*
4557 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4558 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4559 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4560 */
4561 uint16_t u16Sel = 0;
4562 uint32_t u32Limit = 0;
4563 uint64_t u64Base = 0;
4564 uint32_t u32AccessRights = 0;
4565
4566 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4567 {
4568 u16Sel = pMixedCtx->tr.Sel;
4569 u32Limit = pMixedCtx->tr.u32Limit;
4570 u64Base = pMixedCtx->tr.u64Base;
4571 u32AccessRights = pMixedCtx->tr.Attr.u;
4572 }
4573 else
4574 {
4575 Assert(pVM->hm.s.vmx.pRealModeTSS);
4576 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4577
4578 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4579 RTGCPHYS GCPhys;
4580 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4581 AssertRCReturn(rc, rc);
4582
4583 X86DESCATTR DescAttr;
4584 DescAttr.u = 0;
4585 DescAttr.n.u1Present = 1;
4586 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4587
4588 u16Sel = 0;
4589 u32Limit = HM_VTX_TSS_SIZE;
4590 u64Base = GCPhys; /* in real-mode phys = virt. */
4591 u32AccessRights = DescAttr.u;
4592 }
4593
4594 /* Validate. */
4595 Assert(!(u16Sel & RT_BIT(2)));
4596 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4597 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4598 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4599 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4600 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4601 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4602 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4603 Assert( (u32Limit & 0xfff) == 0xfff
4604 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4605 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4606 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4607
4608 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4609 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4610 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4611 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4612
4613 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4614 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4615 }
4616
4617 /*
4618 * Guest GDTR.
4619 */
4620 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4621 {
4622 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4623 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4624
4625 /* Validate. */
4626 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4627
4628 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4629 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4630 }
4631
4632 /*
4633 * Guest LDTR.
4634 */
4635 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4636 {
4637 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4638 uint32_t u32Access = 0;
4639 if (!pMixedCtx->ldtr.Attr.u)
4640 u32Access = X86DESCATTR_UNUSABLE;
4641 else
4642 u32Access = pMixedCtx->ldtr.Attr.u;
4643
4644 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4645 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4646 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4647 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4648
4649 /* Validate. */
4650 if (!(u32Access & X86DESCATTR_UNUSABLE))
4651 {
4652 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4655 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4656 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4657 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4658 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4659 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4660 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4661 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4662 }
4663
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4665 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4666 }
4667
4668 /*
4669 * Guest IDTR.
4670 */
4671 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4672 {
4673 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4674 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4675
4676 /* Validate. */
4677 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4678
4679 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4680 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4681 }
4682
4683 return VINF_SUCCESS;
4684}
4685
4686
4687/**
4688 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4689 * areas. These MSRs will automatically be loaded to the host CPU on every
4690 * successful VM-entry and stored from the host CPU on every successful VM-exit.
4691 *
4692 * This also creates/updates MSR slots for the host MSRs. The actual host
4693 * MSR values are -not- updated here for performance reasons. See
4694 * hmR0VmxSaveHostMsrs().
4695 *
4696 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4697 *
4698 * @returns VBox status code.
4699 * @param pVCpu Pointer to the VMCPU.
4700 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4701 * out-of-sync. Make sure to update the required fields
4702 * before using them.
4703 *
4704 * @remarks No-long-jump zone!!!
4705 */
4706static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4707{
4708 AssertPtr(pVCpu);
4709 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4710
4711 /*
4712 * MSRs that we use the auto-load/store MSR area in the VMCS.
4713 */
4714 PVM pVM = pVCpu->CTX_SUFF(pVM);
4715 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4716 {
4717 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4718#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4719 if (pVM->hm.s.fAllow64BitGuests)
4720 {
4721 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4722 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4723 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4724 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4725# ifdef DEBUG
4726 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4727 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4728 {
4729 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4730 pMsr->u64Value));
4731 }
4732# endif
4733 }
4734#endif
4735 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4736 }
4737
4738 /*
4739 * Guest Sysenter MSRs.
4740 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4741 * VM-exits on WRMSRs for these MSRs.
4742 */
4743 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4744 {
4745 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4746 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4747 }
4748
4749 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4750 {
4751 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4752 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4753 }
4754
4755 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4756 {
4757 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4758 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4759 }
4760
4761 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4762 {
4763 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4764 {
4765 /*
4766 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4767 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4768 */
4769 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4770 {
4771 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4772 AssertRCReturn(rc,rc);
4773 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4774 }
4775 else
4776 {
4777 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4778 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4779 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4780 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4781 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4782 }
4783 }
4784 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4785 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4786 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4787 }
4788
4789 return VINF_SUCCESS;
4790}
4791
4792
4793/**
4794 * Loads the guest activity state into the guest-state area in the VMCS.
4795 *
4796 * @returns VBox status code.
4797 * @param pVCpu Pointer to the VMCPU.
4798 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4799 * out-of-sync. Make sure to update the required fields
4800 * before using them.
4801 *
4802 * @remarks No-long-jump zone!!!
4803 */
4804static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4805{
4806 NOREF(pCtx);
4807 /** @todo See if we can make use of other states, e.g.
4808 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4809 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4810 {
4811 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4812 AssertRCReturn(rc, rc);
4813
4814 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4815 }
4816 return VINF_SUCCESS;
4817}
4818
4819
4820/**
4821 * Sets up the appropriate function to run guest code.
4822 *
4823 * @returns VBox status code.
4824 * @param pVCpu Pointer to the VMCPU.
4825 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4826 * out-of-sync. Make sure to update the required fields
4827 * before using them.
4828 *
4829 * @remarks No-long-jump zone!!!
4830 */
4831static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4832{
4833 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4834 {
4835#ifndef VBOX_ENABLE_64_BITS_GUESTS
4836 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4837#endif
4838 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4839#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4840 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4841 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4842 {
4843 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4844 {
4845 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4846 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4847 | HM_CHANGED_VMX_EXIT_CTLS
4848 | HM_CHANGED_VMX_ENTRY_CTLS
4849 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4850 }
4851 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4852 }
4853#else
4854 /* 64-bit host or hybrid host. */
4855 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4856#endif
4857 }
4858 else
4859 {
4860 /* Guest is not in long mode, use the 32-bit handler. */
4861#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4862 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4863 {
4864 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4865 {
4866 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4867 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4868 | HM_CHANGED_VMX_EXIT_CTLS
4869 | HM_CHANGED_VMX_ENTRY_CTLS
4870 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4871 }
4872 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4873 }
4874#else
4875 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4876#endif
4877 }
4878 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4879 return VINF_SUCCESS;
4880}
4881
4882
4883/**
4884 * Wrapper for running the guest code in VT-x.
4885 *
4886 * @returns VBox strict status code.
4887 * @param pVM Pointer to the VM.
4888 * @param pVCpu Pointer to the VMCPU.
4889 * @param pCtx Pointer to the guest-CPU context.
4890 *
4891 * @remarks No-long-jump zone!!!
4892 */
4893DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4894{
4895 /*
4896 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4897 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4898 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4899 */
4900 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4901 /** @todo Add stats for resume vs launch. */
4902#ifdef VBOX_WITH_KERNEL_USING_XMM
4903 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4904#else
4905 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4906#endif
4907}
4908
4909
4910/**
4911 * Reports world-switch error and dumps some useful debug info.
4912 *
4913 * @param pVM Pointer to the VM.
4914 * @param pVCpu Pointer to the VMCPU.
4915 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4916 * @param pCtx Pointer to the guest-CPU context.
4917 * @param pVmxTransient Pointer to the VMX transient structure (only
4918 * exitReason updated).
4919 */
4920static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4921{
4922 Assert(pVM);
4923 Assert(pVCpu);
4924 Assert(pCtx);
4925 Assert(pVmxTransient);
4926 HMVMX_ASSERT_PREEMPT_SAFE();
4927
4928 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4929 switch (rcVMRun)
4930 {
4931 case VERR_VMX_INVALID_VMXON_PTR:
4932 AssertFailed();
4933 break;
4934 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4935 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4936 {
4937 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4938 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4939 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4940 AssertRC(rc);
4941
4942 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4943 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4944 Cannot do it here as we may have been long preempted. */
4945
4946#ifdef VBOX_STRICT
4947 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4948 pVmxTransient->uExitReason));
4949 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4950 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4951 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4952 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4953 else
4954 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4955 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4956 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4957
4958 /* VMX control bits. */
4959 uint32_t u32Val;
4960 uint64_t u64Val;
4961 HMVMXHCUINTREG uHCReg;
4962 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4963 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4964 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4965 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4970 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4971 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4974 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4975 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4976 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4977 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4978 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4979 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4980 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4981 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4982 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4983 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4984 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4985 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4986 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4987 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4992 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4993 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4994 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4995 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4996 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4997 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4998 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4999 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5000 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5001 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5002 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5003 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5004
5005 /* Guest bits. */
5006 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5007 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5008 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5009 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5010 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5011 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5012 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5013 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5014
5015 /* Host bits. */
5016 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5017 Log4(("Host CR0 %#RHr\n", uHCReg));
5018 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5019 Log4(("Host CR3 %#RHr\n", uHCReg));
5020 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5021 Log4(("Host CR4 %#RHr\n", uHCReg));
5022
5023 RTGDTR HostGdtr;
5024 PCX86DESCHC pDesc;
5025 ASMGetGDTR(&HostGdtr);
5026 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5027 Log4(("Host CS %#08x\n", u32Val));
5028 if (u32Val < HostGdtr.cbGdt)
5029 {
5030 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5031 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5032 }
5033
5034 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5035 Log4(("Host DS %#08x\n", u32Val));
5036 if (u32Val < HostGdtr.cbGdt)
5037 {
5038 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5039 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5040 }
5041
5042 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5043 Log4(("Host ES %#08x\n", u32Val));
5044 if (u32Val < HostGdtr.cbGdt)
5045 {
5046 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5047 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5048 }
5049
5050 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5051 Log4(("Host FS %#08x\n", u32Val));
5052 if (u32Val < HostGdtr.cbGdt)
5053 {
5054 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5055 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5056 }
5057
5058 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5059 Log4(("Host GS %#08x\n", u32Val));
5060 if (u32Val < HostGdtr.cbGdt)
5061 {
5062 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5063 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5064 }
5065
5066 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5067 Log4(("Host SS %#08x\n", u32Val));
5068 if (u32Val < HostGdtr.cbGdt)
5069 {
5070 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5071 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5072 }
5073
5074 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5075 Log4(("Host TR %#08x\n", u32Val));
5076 if (u32Val < HostGdtr.cbGdt)
5077 {
5078 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5079 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5080 }
5081
5082 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5083 Log4(("Host TR Base %#RHv\n", uHCReg));
5084 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5085 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5086 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5087 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5088 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5089 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5091 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5092 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5093 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5095 Log4(("Host RSP %#RHv\n", uHCReg));
5096 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5097 Log4(("Host RIP %#RHv\n", uHCReg));
5098# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5099 if (HMVMX_IS_64BIT_HOST_MODE())
5100 {
5101 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5102 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5103 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5104 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5105 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5106 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5107 }
5108# endif
5109#endif /* VBOX_STRICT */
5110 break;
5111 }
5112
5113 default:
5114 /* Impossible */
5115 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5116 break;
5117 }
5118 NOREF(pVM); NOREF(pCtx);
5119}
5120
5121
5122#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5123#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5124# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5125#endif
5126#ifdef VBOX_STRICT
5127static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5128{
5129 switch (idxField)
5130 {
5131 case VMX_VMCS_GUEST_RIP:
5132 case VMX_VMCS_GUEST_RSP:
5133 case VMX_VMCS_GUEST_SYSENTER_EIP:
5134 case VMX_VMCS_GUEST_SYSENTER_ESP:
5135 case VMX_VMCS_GUEST_GDTR_BASE:
5136 case VMX_VMCS_GUEST_IDTR_BASE:
5137 case VMX_VMCS_GUEST_CS_BASE:
5138 case VMX_VMCS_GUEST_DS_BASE:
5139 case VMX_VMCS_GUEST_ES_BASE:
5140 case VMX_VMCS_GUEST_FS_BASE:
5141 case VMX_VMCS_GUEST_GS_BASE:
5142 case VMX_VMCS_GUEST_SS_BASE:
5143 case VMX_VMCS_GUEST_LDTR_BASE:
5144 case VMX_VMCS_GUEST_TR_BASE:
5145 case VMX_VMCS_GUEST_CR3:
5146 return true;
5147 }
5148 return false;
5149}
5150
5151static bool hmR0VmxIsValidReadField(uint32_t idxField)
5152{
5153 switch (idxField)
5154 {
5155 /* Read-only fields. */
5156 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5157 return true;
5158 }
5159 /* Remaining readable fields should also be writable. */
5160 return hmR0VmxIsValidWriteField(idxField);
5161}
5162#endif /* VBOX_STRICT */
5163
5164
5165/**
5166 * Executes the specified handler in 64-bit mode.
5167 *
5168 * @returns VBox status code.
5169 * @param pVM Pointer to the VM.
5170 * @param pVCpu Pointer to the VMCPU.
5171 * @param pCtx Pointer to the guest CPU context.
5172 * @param enmOp The operation to perform.
5173 * @param cbParam Number of parameters.
5174 * @param paParam Array of 32-bit parameters.
5175 */
5176VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5177 uint32_t *paParam)
5178{
5179 int rc, rc2;
5180 PHMGLOBALCPUINFO pCpu;
5181 RTHCPHYS HCPhysCpuPage;
5182 RTCCUINTREG uOldEflags;
5183
5184 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5185 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5186 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5187 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5188
5189#ifdef VBOX_STRICT
5190 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5191 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5192
5193 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5194 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5195#endif
5196
5197 /* Disable interrupts. */
5198 uOldEflags = ASMIntDisableFlags();
5199
5200#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5201 RTCPUID idHostCpu = RTMpCpuId();
5202 CPUMR0SetLApic(pVCpu, idHostCpu);
5203#endif
5204
5205 pCpu = HMR0GetCurrentCpu();
5206 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5207
5208 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5209 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5210
5211 /* Leave VMX Root Mode. */
5212 VMXDisable();
5213
5214 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5215
5216 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5217 CPUMSetHyperEIP(pVCpu, enmOp);
5218 for (int i = (int)cbParam - 1; i >= 0; i--)
5219 CPUMPushHyper(pVCpu, paParam[i]);
5220
5221 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5222
5223 /* Call the switcher. */
5224 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5225 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5226
5227 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5228 /* Make sure the VMX instructions don't cause #UD faults. */
5229 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5230
5231 /* Re-enter VMX Root Mode */
5232 rc2 = VMXEnable(HCPhysCpuPage);
5233 if (RT_FAILURE(rc2))
5234 {
5235 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5236 ASMSetFlags(uOldEflags);
5237 return rc2;
5238 }
5239
5240 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5241 AssertRC(rc2);
5242 Assert(!(ASMGetFlags() & X86_EFL_IF));
5243 ASMSetFlags(uOldEflags);
5244 return rc;
5245}
5246
5247
5248/**
5249 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5250 * supporting 64-bit guests.
5251 *
5252 * @returns VBox status code.
5253 * @param fResume Whether to VMLAUNCH or VMRESUME.
5254 * @param pCtx Pointer to the guest-CPU context.
5255 * @param pCache Pointer to the VMCS cache.
5256 * @param pVM Pointer to the VM.
5257 * @param pVCpu Pointer to the VMCPU.
5258 */
5259DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5260{
5261 uint32_t aParam[6];
5262 PHMGLOBALCPUINFO pCpu = NULL;
5263 RTHCPHYS HCPhysCpuPage = 0;
5264 int rc = VERR_INTERNAL_ERROR_5;
5265
5266 pCpu = HMR0GetCurrentCpu();
5267 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5268
5269#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5270 pCache->uPos = 1;
5271 pCache->interPD = PGMGetInterPaeCR3(pVM);
5272 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5273#endif
5274
5275#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5276 pCache->TestIn.HCPhysCpuPage = 0;
5277 pCache->TestIn.HCPhysVmcs = 0;
5278 pCache->TestIn.pCache = 0;
5279 pCache->TestOut.HCPhysVmcs = 0;
5280 pCache->TestOut.pCache = 0;
5281 pCache->TestOut.pCtx = 0;
5282 pCache->TestOut.eflags = 0;
5283#endif
5284
5285 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5286 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5287 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5288 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5289 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5290 aParam[5] = 0;
5291
5292#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5293 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5294 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5295#endif
5296 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5297
5298#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5299 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5300 Assert(pCtx->dr[4] == 10);
5301 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5302#endif
5303
5304#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5305 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5306 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5307 pVCpu->hm.s.vmx.HCPhysVmcs));
5308 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5309 pCache->TestOut.HCPhysVmcs));
5310 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5311 pCache->TestOut.pCache));
5312 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5313 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5314 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5315 pCache->TestOut.pCtx));
5316 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5317#endif
5318 return rc;
5319}
5320
5321
5322/**
5323 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5324 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5325 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5326 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5327 *
5328 * @returns VBox status code.
5329 * @param pVM Pointer to the VM.
5330 * @param pVCpu Pointer to the VMCPU.
5331 */
5332static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5333{
5334#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5335{ \
5336 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5337 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5338 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5339 ++cReadFields; \
5340}
5341
5342 AssertPtr(pVM);
5343 AssertPtr(pVCpu);
5344 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5345 uint32_t cReadFields = 0;
5346
5347 /*
5348 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5349 * and serve to indicate exceptions to the rules.
5350 */
5351
5352 /* Guest-natural selector base fields. */
5353#if 0
5354 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5355 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5357#endif
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5368 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5370#if 0
5371 /* Unused natural width guest-state fields. */
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5374#endif
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5377
5378 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5379#if 0
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5389#endif
5390
5391 /* Natural width guest-state fields. */
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5393#if 0
5394 /* Currently unused field. */
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5396#endif
5397
5398 if (pVM->hm.s.fNestedPaging)
5399 {
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5401 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5402 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5403 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5404 }
5405 else
5406 {
5407 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5408 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5409 }
5410
5411#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5412 return VINF_SUCCESS;
5413}
5414
5415
5416/**
5417 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5418 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5419 * darwin, running 64-bit guests).
5420 *
5421 * @returns VBox status code.
5422 * @param pVCpu Pointer to the VMCPU.
5423 * @param idxField The VMCS field encoding.
5424 * @param u64Val 16, 32 or 64-bit value.
5425 */
5426VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5427{
5428 int rc;
5429 switch (idxField)
5430 {
5431 /*
5432 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5433 */
5434 /* 64-bit Control fields. */
5435 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5436 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5437 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5438 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5439 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5440 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5441 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5442 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5443 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5444 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5445 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5446 case VMX_VMCS64_CTRL_EPTP_FULL:
5447 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5448 /* 64-bit Guest-state fields. */
5449 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5450 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5451 case VMX_VMCS64_GUEST_PAT_FULL:
5452 case VMX_VMCS64_GUEST_EFER_FULL:
5453 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5454 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5455 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5456 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5457 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5458 /* 64-bit Host-state fields. */
5459 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5460 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5461 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5462 {
5463 rc = VMXWriteVmcs32(idxField, u64Val);
5464 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5465 break;
5466 }
5467
5468 /*
5469 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5470 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5471 */
5472 /* Natural-width Guest-state fields. */
5473 case VMX_VMCS_GUEST_CR3:
5474 case VMX_VMCS_GUEST_ES_BASE:
5475 case VMX_VMCS_GUEST_CS_BASE:
5476 case VMX_VMCS_GUEST_SS_BASE:
5477 case VMX_VMCS_GUEST_DS_BASE:
5478 case VMX_VMCS_GUEST_FS_BASE:
5479 case VMX_VMCS_GUEST_GS_BASE:
5480 case VMX_VMCS_GUEST_LDTR_BASE:
5481 case VMX_VMCS_GUEST_TR_BASE:
5482 case VMX_VMCS_GUEST_GDTR_BASE:
5483 case VMX_VMCS_GUEST_IDTR_BASE:
5484 case VMX_VMCS_GUEST_RSP:
5485 case VMX_VMCS_GUEST_RIP:
5486 case VMX_VMCS_GUEST_SYSENTER_ESP:
5487 case VMX_VMCS_GUEST_SYSENTER_EIP:
5488 {
5489 if (!(u64Val >> 32))
5490 {
5491 /* If this field is 64-bit, VT-x will zero out the top bits. */
5492 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5493 }
5494 else
5495 {
5496 /* Assert that only the 32->64 switcher case should ever come here. */
5497 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5498 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5499 }
5500 break;
5501 }
5502
5503 default:
5504 {
5505 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5506 rc = VERR_INVALID_PARAMETER;
5507 break;
5508 }
5509 }
5510 AssertRCReturn(rc, rc);
5511 return rc;
5512}
5513
5514
5515/**
5516 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5517 * hosts (except darwin) for 64-bit guests.
5518 *
5519 * @param pVCpu Pointer to the VMCPU.
5520 * @param idxField The VMCS field encoding.
5521 * @param u64Val 16, 32 or 64-bit value.
5522 */
5523VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5524{
5525 AssertPtr(pVCpu);
5526 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5527
5528 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5529 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5530
5531 /* Make sure there are no duplicates. */
5532 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5533 {
5534 if (pCache->Write.aField[i] == idxField)
5535 {
5536 pCache->Write.aFieldVal[i] = u64Val;
5537 return VINF_SUCCESS;
5538 }
5539 }
5540
5541 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5542 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5543 pCache->Write.cValidEntries++;
5544 return VINF_SUCCESS;
5545}
5546
5547/* Enable later when the assembly code uses these as callbacks. */
5548#if 0
5549/*
5550 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5551 *
5552 * @param pVCpu Pointer to the VMCPU.
5553 * @param pCache Pointer to the VMCS cache.
5554 *
5555 * @remarks No-long-jump zone!!!
5556 */
5557VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5558{
5559 AssertPtr(pCache);
5560 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5561 {
5562 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5563 AssertRC(rc);
5564 }
5565 pCache->Write.cValidEntries = 0;
5566}
5567
5568
5569/**
5570 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5571 *
5572 * @param pVCpu Pointer to the VMCPU.
5573 * @param pCache Pointer to the VMCS cache.
5574 *
5575 * @remarks No-long-jump zone!!!
5576 */
5577VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5578{
5579 AssertPtr(pCache);
5580 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5581 {
5582 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5583 AssertRC(rc);
5584 }
5585}
5586#endif
5587#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5588
5589
5590/**
5591 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5592 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5593 * timer.
5594 *
5595 * @returns VBox status code.
5596 * @param pVCpu Pointer to the VMCPU.
5597 *
5598 * @remarks No-long-jump zone!!!
5599 */
5600static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5601{
5602 int rc = VERR_INTERNAL_ERROR_5;
5603 bool fOffsettedTsc = false;
5604 bool fParavirtTsc = false;
5605 PVM pVM = pVCpu->CTX_SUFF(pVM);
5606 if (pVM->hm.s.vmx.fUsePreemptTimer)
5607 {
5608 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fOffsettedTsc,
5609 &fParavirtTsc);
5610
5611 /* Make sure the returned values have sane upper and lower boundaries. */
5612 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5613 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5614 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5615 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5616
5617 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5618 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5619 }
5620 else
5621 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5622
5623 /** @todo later optimize this to be done elsewhere and not before every
5624 * VM-entry. */
5625 if (fParavirtTsc)
5626 {
5627 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5628 AssertRC(rc);
5629 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5630 }
5631
5632 if (fOffsettedTsc)
5633 {
5634 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5635 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5636
5637 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5638 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5639 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5640 }
5641 else
5642 {
5643 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5644 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5645 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5646 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5647 }
5648}
5649
5650
5651/**
5652 * Determines if an exception is a contributory exception. Contributory
5653 * exceptions are ones which can cause double-faults. Page-fault is
5654 * intentionally not included here as it's a conditional contributory exception.
5655 *
5656 * @returns true if the exception is contributory, false otherwise.
5657 * @param uVector The exception vector.
5658 */
5659DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5660{
5661 switch (uVector)
5662 {
5663 case X86_XCPT_GP:
5664 case X86_XCPT_SS:
5665 case X86_XCPT_NP:
5666 case X86_XCPT_TS:
5667 case X86_XCPT_DE:
5668 return true;
5669 default:
5670 break;
5671 }
5672 return false;
5673}
5674
5675
5676/**
5677 * Sets an event as a pending event to be injected into the guest.
5678 *
5679 * @param pVCpu Pointer to the VMCPU.
5680 * @param u32IntInfo The VM-entry interruption-information field.
5681 * @param cbInstr The VM-entry instruction length in bytes (for software
5682 * interrupts, exceptions and privileged software
5683 * exceptions).
5684 * @param u32ErrCode The VM-entry exception error code.
5685 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5686 * page-fault.
5687 *
5688 * @remarks Statistics counter assumes this is a guest event being injected or
5689 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5690 * always incremented.
5691 */
5692DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5693 RTGCUINTPTR GCPtrFaultAddress)
5694{
5695 Assert(!pVCpu->hm.s.Event.fPending);
5696 pVCpu->hm.s.Event.fPending = true;
5697 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5698 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5699 pVCpu->hm.s.Event.cbInstr = cbInstr;
5700 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5701
5702 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5703}
5704
5705
5706/**
5707 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5708 *
5709 * @param pVCpu Pointer to the VMCPU.
5710 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5711 * out-of-sync. Make sure to update the required fields
5712 * before using them.
5713 */
5714DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5715{
5716 NOREF(pMixedCtx);
5717 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5718 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5719 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5720 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5721}
5722
5723
5724/**
5725 * Handle a condition that occurred while delivering an event through the guest
5726 * IDT.
5727 *
5728 * @returns VBox status code (informational error codes included).
5729 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5730 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5731 * continue execution of the guest which will delivery the #DF.
5732 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5733 *
5734 * @param pVCpu Pointer to the VMCPU.
5735 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5736 * out-of-sync. Make sure to update the required fields
5737 * before using them.
5738 * @param pVmxTransient Pointer to the VMX transient structure.
5739 *
5740 * @remarks No-long-jump zone!!!
5741 */
5742static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5743{
5744 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5745
5746 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5747 AssertRCReturn(rc, rc);
5748 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5749 AssertRCReturn(rc, rc);
5750
5751 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5752 {
5753 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5754 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5755
5756 typedef enum
5757 {
5758 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5759 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5760 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5761 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5762 } VMXREFLECTXCPT;
5763
5764 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5765 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5766 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5767 {
5768 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5769 {
5770 enmReflect = VMXREFLECTXCPT_XCPT;
5771#ifdef VBOX_STRICT
5772 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5773 && uExitVector == X86_XCPT_PF)
5774 {
5775 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5776 }
5777#endif
5778 if ( uExitVector == X86_XCPT_PF
5779 && uIdtVector == X86_XCPT_PF)
5780 {
5781 pVmxTransient->fVectoringDoublePF = true;
5782 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5783 }
5784 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5785 && hmR0VmxIsContributoryXcpt(uExitVector)
5786 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5787 || uIdtVector == X86_XCPT_PF))
5788 {
5789 enmReflect = VMXREFLECTXCPT_DF;
5790 }
5791 else if (uIdtVector == X86_XCPT_DF)
5792 enmReflect = VMXREFLECTXCPT_TF;
5793 }
5794 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5795 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5796 {
5797 /*
5798 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5799 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5800 */
5801 enmReflect = VMXREFLECTXCPT_XCPT;
5802
5803 if (uExitVector == X86_XCPT_PF)
5804 {
5805 pVmxTransient->fVectoringPF = true;
5806 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5807 }
5808 }
5809 }
5810 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5811 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5812 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5813 {
5814 /*
5815 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5816 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5817 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5818 */
5819 enmReflect = VMXREFLECTXCPT_XCPT;
5820 }
5821
5822 /*
5823 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5824 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5825 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5826 *
5827 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5828 */
5829 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5830 && enmReflect == VMXREFLECTXCPT_XCPT
5831 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5832 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5833 {
5834 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5835 }
5836
5837 switch (enmReflect)
5838 {
5839 case VMXREFLECTXCPT_XCPT:
5840 {
5841 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5842 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5843 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5844
5845 uint32_t u32ErrCode = 0;
5846 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5847 {
5848 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5849 AssertRCReturn(rc, rc);
5850 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5851 }
5852
5853 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5854 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5855 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5856 rc = VINF_SUCCESS;
5857 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5858 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5859
5860 break;
5861 }
5862
5863 case VMXREFLECTXCPT_DF:
5864 {
5865 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5866 rc = VINF_HM_DOUBLE_FAULT;
5867 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5868 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5869
5870 break;
5871 }
5872
5873 case VMXREFLECTXCPT_TF:
5874 {
5875 rc = VINF_EM_RESET;
5876 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5877 uExitVector));
5878 break;
5879 }
5880
5881 default:
5882 Assert(rc == VINF_SUCCESS);
5883 break;
5884 }
5885 }
5886 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5887 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5888 && uExitVector != X86_XCPT_DF
5889 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5890 {
5891 /*
5892 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5893 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5894 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5895 */
5896 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5897 {
5898 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5899 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5900 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5901 }
5902 }
5903
5904 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5905 return rc;
5906}
5907
5908
5909/**
5910 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5911 *
5912 * @returns VBox status code.
5913 * @param pVCpu Pointer to the VMCPU.
5914 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5915 * out-of-sync. Make sure to update the required fields
5916 * before using them.
5917 *
5918 * @remarks No-long-jump zone!!!
5919 */
5920static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5921{
5922 NOREF(pMixedCtx);
5923
5924 /*
5925 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5926 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5927 */
5928 VMMRZCallRing3Disable(pVCpu);
5929 HM_DISABLE_PREEMPT_IF_NEEDED();
5930
5931 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5932 {
5933 uint32_t uVal = 0;
5934 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5935 AssertRCReturn(rc, rc);
5936
5937 uint32_t uShadow = 0;
5938 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5939 AssertRCReturn(rc, rc);
5940
5941 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5942 CPUMSetGuestCR0(pVCpu, uVal);
5943 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5944 }
5945
5946 HM_RESTORE_PREEMPT_IF_NEEDED();
5947 VMMRZCallRing3Enable(pVCpu);
5948 return VINF_SUCCESS;
5949}
5950
5951
5952/**
5953 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5954 *
5955 * @returns VBox status code.
5956 * @param pVCpu Pointer to the VMCPU.
5957 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5958 * out-of-sync. Make sure to update the required fields
5959 * before using them.
5960 *
5961 * @remarks No-long-jump zone!!!
5962 */
5963static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5964{
5965 NOREF(pMixedCtx);
5966
5967 int rc = VINF_SUCCESS;
5968 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5969 {
5970 uint32_t uVal = 0;
5971 uint32_t uShadow = 0;
5972 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5973 AssertRCReturn(rc, rc);
5974 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5975 AssertRCReturn(rc, rc);
5976
5977 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5978 CPUMSetGuestCR4(pVCpu, uVal);
5979 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5980 }
5981 return rc;
5982}
5983
5984
5985/**
5986 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5987 *
5988 * @returns VBox status code.
5989 * @param pVCpu Pointer to the VMCPU.
5990 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5991 * out-of-sync. Make sure to update the required fields
5992 * before using them.
5993 *
5994 * @remarks No-long-jump zone!!!
5995 */
5996static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5997{
5998 int rc = VINF_SUCCESS;
5999 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6000 {
6001 uint64_t u64Val = 0;
6002 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6003 AssertRCReturn(rc, rc);
6004
6005 pMixedCtx->rip = u64Val;
6006 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6007 }
6008 return rc;
6009}
6010
6011
6012/**
6013 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6014 *
6015 * @returns VBox status code.
6016 * @param pVCpu Pointer to the VMCPU.
6017 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6018 * out-of-sync. Make sure to update the required fields
6019 * before using them.
6020 *
6021 * @remarks No-long-jump zone!!!
6022 */
6023static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6024{
6025 int rc = VINF_SUCCESS;
6026 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6027 {
6028 uint64_t u64Val = 0;
6029 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6030 AssertRCReturn(rc, rc);
6031
6032 pMixedCtx->rsp = u64Val;
6033 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6034 }
6035 return rc;
6036}
6037
6038
6039/**
6040 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6041 *
6042 * @returns VBox status code.
6043 * @param pVCpu Pointer to the VMCPU.
6044 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6045 * out-of-sync. Make sure to update the required fields
6046 * before using them.
6047 *
6048 * @remarks No-long-jump zone!!!
6049 */
6050static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6051{
6052 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6053 {
6054 uint32_t uVal = 0;
6055 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6056 AssertRCReturn(rc, rc);
6057
6058 pMixedCtx->eflags.u32 = uVal;
6059 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6060 {
6061 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6062 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6063
6064 pMixedCtx->eflags.Bits.u1VM = 0;
6065 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6066 }
6067
6068 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6069 }
6070 return VINF_SUCCESS;
6071}
6072
6073
6074/**
6075 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6076 * guest-CPU context.
6077 */
6078DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6079{
6080 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6081 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6082 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6083 return rc;
6084}
6085
6086
6087/**
6088 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6089 * from the guest-state area in the VMCS.
6090 *
6091 * @param pVCpu Pointer to the VMCPU.
6092 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6093 * out-of-sync. Make sure to update the required fields
6094 * before using them.
6095 *
6096 * @remarks No-long-jump zone!!!
6097 */
6098static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6099{
6100 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6101 {
6102 uint32_t uIntrState = 0;
6103 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6104 AssertRC(rc);
6105
6106 if (!uIntrState)
6107 {
6108 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6109 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6110
6111 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6112 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6113 }
6114 else
6115 {
6116 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6117 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6118 {
6119 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6120 AssertRC(rc);
6121 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6122 AssertRC(rc);
6123
6124 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6125 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6126 }
6127 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6128 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6129
6130 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6131 {
6132 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6133 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6134 }
6135 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6136 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6137 }
6138
6139 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6140 }
6141}
6142
6143
6144/**
6145 * Saves the guest's activity state.
6146 *
6147 * @returns VBox status code.
6148 * @param pVCpu Pointer to the VMCPU.
6149 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6150 * out-of-sync. Make sure to update the required fields
6151 * before using them.
6152 *
6153 * @remarks No-long-jump zone!!!
6154 */
6155static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6156{
6157 NOREF(pMixedCtx);
6158 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6159 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6160 return VINF_SUCCESS;
6161}
6162
6163
6164/**
6165 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6166 * the current VMCS into the guest-CPU context.
6167 *
6168 * @returns VBox status code.
6169 * @param pVCpu Pointer to the VMCPU.
6170 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6171 * out-of-sync. Make sure to update the required fields
6172 * before using them.
6173 *
6174 * @remarks No-long-jump zone!!!
6175 */
6176static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6177{
6178 int rc = VINF_SUCCESS;
6179 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6180 {
6181 uint32_t u32Val = 0;
6182 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6183 pMixedCtx->SysEnter.cs = u32Val;
6184 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6185 }
6186
6187 uint64_t u64Val = 0;
6188 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6189 {
6190 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6191 pMixedCtx->SysEnter.eip = u64Val;
6192 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6193 }
6194 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6195 {
6196 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6197 pMixedCtx->SysEnter.esp = u64Val;
6198 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6199 }
6200 return rc;
6201}
6202
6203
6204/**
6205 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6206 * the CPU back into the guest-CPU context.
6207 *
6208 * @returns VBox status code.
6209 * @param pVCpu Pointer to the VMCPU.
6210 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6211 * out-of-sync. Make sure to update the required fields
6212 * before using them.
6213 *
6214 * @remarks No-long-jump zone!!!
6215 */
6216static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6217{
6218#if HC_ARCH_BITS == 64
6219 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6220 {
6221 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6222 VMMRZCallRing3Disable(pVCpu);
6223 HM_DISABLE_PREEMPT_IF_NEEDED();
6224
6225 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6226 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6227 {
6228 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6229 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6230 }
6231
6232 HM_RESTORE_PREEMPT_IF_NEEDED();
6233 VMMRZCallRing3Enable(pVCpu);
6234 }
6235 else
6236 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6237#else
6238 NOREF(pMixedCtx);
6239 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6240#endif
6241
6242 return VINF_SUCCESS;
6243}
6244
6245
6246/**
6247 * Saves the auto load/store'd guest MSRs from the current VMCS into
6248 * the guest-CPU context.
6249 *
6250 * @returns VBox status code.
6251 * @param pVCpu Pointer to the VMCPU.
6252 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6253 * out-of-sync. Make sure to update the required fields
6254 * before using them.
6255 *
6256 * @remarks No-long-jump zone!!!
6257 */
6258static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6259{
6260 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6261 return VINF_SUCCESS;
6262
6263 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6264 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6265 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6266 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6267 {
6268 switch (pMsr->u32Msr)
6269 {
6270 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6271 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6272 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6273 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6274 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6275 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6276 break;
6277
6278 default:
6279 {
6280 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6281 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6282 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6283 }
6284 }
6285 }
6286
6287 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6288 return VINF_SUCCESS;
6289}
6290
6291
6292/**
6293 * Saves the guest control registers from the current VMCS into the guest-CPU
6294 * context.
6295 *
6296 * @returns VBox status code.
6297 * @param pVCpu Pointer to the VMCPU.
6298 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6299 * out-of-sync. Make sure to update the required fields
6300 * before using them.
6301 *
6302 * @remarks No-long-jump zone!!!
6303 */
6304static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6305{
6306 /* Guest CR0. Guest FPU. */
6307 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6308 AssertRCReturn(rc, rc);
6309
6310 /* Guest CR4. */
6311 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6312 AssertRCReturn(rc, rc);
6313
6314 /* Guest CR2 - updated always during the world-switch or in #PF. */
6315 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6316 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6317 {
6318 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6319 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6320
6321 PVM pVM = pVCpu->CTX_SUFF(pVM);
6322 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6323 || ( pVM->hm.s.fNestedPaging
6324 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6325 {
6326 uint64_t u64Val = 0;
6327 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6328 if (pMixedCtx->cr3 != u64Val)
6329 {
6330 CPUMSetGuestCR3(pVCpu, u64Val);
6331 if (VMMRZCallRing3IsEnabled(pVCpu))
6332 {
6333 PGMUpdateCR3(pVCpu, u64Val);
6334 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6335 }
6336 else
6337 {
6338 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6339 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6340 }
6341 }
6342
6343 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6344 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6345 {
6346 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6347 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6348 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6349 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6350
6351 if (VMMRZCallRing3IsEnabled(pVCpu))
6352 {
6353 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6354 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6355 }
6356 else
6357 {
6358 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6359 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6360 }
6361 }
6362 }
6363
6364 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6365 }
6366
6367 /*
6368 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6369 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6370 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6371 *
6372 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6373 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6374 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6375 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6376 *
6377 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6378 */
6379 if (VMMRZCallRing3IsEnabled(pVCpu))
6380 {
6381 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6382 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6383
6384 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6385 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6386
6387 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6388 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6389 }
6390
6391 return rc;
6392}
6393
6394
6395/**
6396 * Reads a guest segment register from the current VMCS into the guest-CPU
6397 * context.
6398 *
6399 * @returns VBox status code.
6400 * @param pVCpu Pointer to the VMCPU.
6401 * @param idxSel Index of the selector in the VMCS.
6402 * @param idxLimit Index of the segment limit in the VMCS.
6403 * @param idxBase Index of the segment base in the VMCS.
6404 * @param idxAccess Index of the access rights of the segment in the VMCS.
6405 * @param pSelReg Pointer to the segment selector.
6406 *
6407 * @remarks No-long-jump zone!!!
6408 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6409 * macro as that takes care of whether to read from the VMCS cache or
6410 * not.
6411 */
6412DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6413 PCPUMSELREG pSelReg)
6414{
6415 NOREF(pVCpu);
6416
6417 uint32_t u32Val = 0;
6418 int rc = VMXReadVmcs32(idxSel, &u32Val);
6419 AssertRCReturn(rc, rc);
6420 pSelReg->Sel = (uint16_t)u32Val;
6421 pSelReg->ValidSel = (uint16_t)u32Val;
6422 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6423
6424 rc = VMXReadVmcs32(idxLimit, &u32Val);
6425 AssertRCReturn(rc, rc);
6426 pSelReg->u32Limit = u32Val;
6427
6428 uint64_t u64Val = 0;
6429 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6430 AssertRCReturn(rc, rc);
6431 pSelReg->u64Base = u64Val;
6432
6433 rc = VMXReadVmcs32(idxAccess, &u32Val);
6434 AssertRCReturn(rc, rc);
6435 pSelReg->Attr.u = u32Val;
6436
6437 /*
6438 * If VT-x marks the segment as unusable, most other bits remain undefined:
6439 * - For CS the L, D and G bits have meaning.
6440 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6441 * - For the remaining data segments no bits are defined.
6442 *
6443 * The present bit and the unusable bit has been observed to be set at the
6444 * same time (the selector was supposed to be invalid as we started executing
6445 * a V8086 interrupt in ring-0).
6446 *
6447 * What should be important for the rest of the VBox code, is that the P bit is
6448 * cleared. Some of the other VBox code recognizes the unusable bit, but
6449 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6450 * safe side here, we'll strip off P and other bits we don't care about. If
6451 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6452 *
6453 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6454 */
6455 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6456 {
6457 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6458
6459 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6460 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6461 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6462
6463 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6464#ifdef DEBUG_bird
6465 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6466 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6467 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6468#endif
6469 }
6470 return VINF_SUCCESS;
6471}
6472
6473
6474#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6475# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6476 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6477 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6478#else
6479# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6480 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6481 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6482#endif
6483
6484
6485/**
6486 * Saves the guest segment registers from the current VMCS into the guest-CPU
6487 * context.
6488 *
6489 * @returns VBox status code.
6490 * @param pVCpu Pointer to the VMCPU.
6491 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6492 * out-of-sync. Make sure to update the required fields
6493 * before using them.
6494 *
6495 * @remarks No-long-jump zone!!!
6496 */
6497static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6498{
6499 /* Guest segment registers. */
6500 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6501 {
6502 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6503 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6504 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6505 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6506 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6507 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6508 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6509
6510 /* Restore segment attributes for real-on-v86 mode hack. */
6511 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6512 {
6513 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6514 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6515 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6516 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6517 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6518 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6519 }
6520 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6521 }
6522
6523 return VINF_SUCCESS;
6524}
6525
6526
6527/**
6528 * Saves the guest descriptor table registers and task register from the current
6529 * VMCS into the guest-CPU context.
6530 *
6531 * @returns VBox status code.
6532 * @param pVCpu Pointer to the VMCPU.
6533 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6534 * out-of-sync. Make sure to update the required fields
6535 * before using them.
6536 *
6537 * @remarks No-long-jump zone!!!
6538 */
6539static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6540{
6541 int rc = VINF_SUCCESS;
6542
6543 /* Guest LDTR. */
6544 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6545 {
6546 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6547 AssertRCReturn(rc, rc);
6548 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6549 }
6550
6551 /* Guest GDTR. */
6552 uint64_t u64Val = 0;
6553 uint32_t u32Val = 0;
6554 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6555 {
6556 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6557 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6558 pMixedCtx->gdtr.pGdt = u64Val;
6559 pMixedCtx->gdtr.cbGdt = u32Val;
6560 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6561 }
6562
6563 /* Guest IDTR. */
6564 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6565 {
6566 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6567 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6568 pMixedCtx->idtr.pIdt = u64Val;
6569 pMixedCtx->idtr.cbIdt = u32Val;
6570 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6571 }
6572
6573 /* Guest TR. */
6574 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6575 {
6576 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6577 AssertRCReturn(rc, rc);
6578
6579 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6580 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6581 {
6582 rc = VMXLOCAL_READ_SEG(TR, tr);
6583 AssertRCReturn(rc, rc);
6584 }
6585 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6586 }
6587 return rc;
6588}
6589
6590#undef VMXLOCAL_READ_SEG
6591
6592
6593/**
6594 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6595 * context.
6596 *
6597 * @returns VBox status code.
6598 * @param pVCpu Pointer to the VMCPU.
6599 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6600 * out-of-sync. Make sure to update the required fields
6601 * before using them.
6602 *
6603 * @remarks No-long-jump zone!!!
6604 */
6605static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6606{
6607 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6608 {
6609 if (!pVCpu->hm.s.fUsingHyperDR7)
6610 {
6611 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6612 uint32_t u32Val;
6613 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6614 pMixedCtx->dr[7] = u32Val;
6615 }
6616
6617 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6618 }
6619 return VINF_SUCCESS;
6620}
6621
6622
6623/**
6624 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6625 *
6626 * @returns VBox status code.
6627 * @param pVCpu Pointer to the VMCPU.
6628 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6629 * out-of-sync. Make sure to update the required fields
6630 * before using them.
6631 *
6632 * @remarks No-long-jump zone!!!
6633 */
6634static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6635{
6636 NOREF(pMixedCtx);
6637
6638 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6639 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6640 return VINF_SUCCESS;
6641}
6642
6643
6644/**
6645 * Saves the entire guest state from the currently active VMCS into the
6646 * guest-CPU context. This essentially VMREADs all guest-data.
6647 *
6648 * @returns VBox status code.
6649 * @param pVCpu Pointer to the VMCPU.
6650 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6651 * out-of-sync. Make sure to update the required fields
6652 * before using them.
6653 */
6654static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6655{
6656 Assert(pVCpu);
6657 Assert(pMixedCtx);
6658
6659 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6660 return VINF_SUCCESS;
6661
6662 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6663 again on the ring-3 callback path, there is no real need to. */
6664 if (VMMRZCallRing3IsEnabled(pVCpu))
6665 VMMR0LogFlushDisable(pVCpu);
6666 else
6667 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6668 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6669
6670 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6671 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6672
6673 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6674 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6675
6676 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6677 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6678
6679 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6680 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6681
6682 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6683 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6684
6685 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6686 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6687
6688 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6689 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6690
6691 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6692 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6693
6694 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6695 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6696
6697 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6698 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6699
6700 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6701 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6702
6703 if (VMMRZCallRing3IsEnabled(pVCpu))
6704 VMMR0LogFlushEnable(pVCpu);
6705
6706 return rc;
6707}
6708
6709
6710/**
6711 * Check per-VM and per-VCPU force flag actions that require us to go back to
6712 * ring-3 for one reason or another.
6713 *
6714 * @returns VBox status code (information status code included).
6715 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6716 * ring-3.
6717 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6718 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6719 * interrupts)
6720 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6721 * all EMTs to be in ring-3.
6722 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6723 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6724 * to the EM loop.
6725 *
6726 * @param pVM Pointer to the VM.
6727 * @param pVCpu Pointer to the VMCPU.
6728 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6729 * out-of-sync. Make sure to update the required fields
6730 * before using them.
6731 */
6732static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6733{
6734 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6735
6736 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6737 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6738 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6739 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6740 {
6741 /* We need the control registers now, make sure the guest-CPU context is updated. */
6742 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6743 AssertRCReturn(rc3, rc3);
6744
6745 /* Pending HM CR3 sync. */
6746 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6747 {
6748 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6749 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6750 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6751 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6752 }
6753
6754 /* Pending HM PAE PDPEs. */
6755 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6756 {
6757 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6758 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6759 }
6760
6761 /* Pending PGM C3 sync. */
6762 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6763 {
6764 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6765 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6766 if (rc2 != VINF_SUCCESS)
6767 {
6768 AssertRC(rc2);
6769 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6770 return rc2;
6771 }
6772 }
6773
6774 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6775 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6776 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6777 {
6778 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6779 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6780 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6781 return rc2;
6782 }
6783
6784 /* Pending VM request packets, such as hardware interrupts. */
6785 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6786 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6787 {
6788 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6789 return VINF_EM_PENDING_REQUEST;
6790 }
6791
6792 /* Pending PGM pool flushes. */
6793 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6794 {
6795 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6796 return VINF_PGM_POOL_FLUSH_PENDING;
6797 }
6798
6799 /* Pending DMA requests. */
6800 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6801 {
6802 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6803 return VINF_EM_RAW_TO_R3;
6804 }
6805 }
6806
6807 return VINF_SUCCESS;
6808}
6809
6810
6811/**
6812 * Converts any TRPM trap into a pending HM event. This is typically used when
6813 * entering from ring-3 (not longjmp returns).
6814 *
6815 * @param pVCpu Pointer to the VMCPU.
6816 */
6817static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6818{
6819 Assert(TRPMHasTrap(pVCpu));
6820 Assert(!pVCpu->hm.s.Event.fPending);
6821
6822 uint8_t uVector;
6823 TRPMEVENT enmTrpmEvent;
6824 RTGCUINT uErrCode;
6825 RTGCUINTPTR GCPtrFaultAddress;
6826 uint8_t cbInstr;
6827
6828 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6829 AssertRC(rc);
6830
6831 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6832 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6833 if (enmTrpmEvent == TRPM_TRAP)
6834 {
6835 switch (uVector)
6836 {
6837 case X86_XCPT_NMI:
6838 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6839 break;
6840
6841 case X86_XCPT_BP:
6842 case X86_XCPT_OF:
6843 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6844 break;
6845
6846 case X86_XCPT_PF:
6847 case X86_XCPT_DF:
6848 case X86_XCPT_TS:
6849 case X86_XCPT_NP:
6850 case X86_XCPT_SS:
6851 case X86_XCPT_GP:
6852 case X86_XCPT_AC:
6853 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6854 /* no break! */
6855 default:
6856 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6857 break;
6858 }
6859 }
6860 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6861 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6862 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6863 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6864 else
6865 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6866
6867 rc = TRPMResetTrap(pVCpu);
6868 AssertRC(rc);
6869 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6870 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6871
6872 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6873 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6874}
6875
6876
6877/**
6878 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6879 * VT-x to execute any instruction.
6880 *
6881 * @param pvCpu Pointer to the VMCPU.
6882 */
6883static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6884{
6885 Assert(pVCpu->hm.s.Event.fPending);
6886
6887 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6888 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6889 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6890 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6891
6892 /* If a trap was already pending, we did something wrong! */
6893 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6894
6895 TRPMEVENT enmTrapType;
6896 switch (uVectorType)
6897 {
6898 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6899 enmTrapType = TRPM_HARDWARE_INT;
6900 break;
6901
6902 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6903 enmTrapType = TRPM_SOFTWARE_INT;
6904 break;
6905
6906 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6907 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6908 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6909 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6910 enmTrapType = TRPM_TRAP;
6911 break;
6912
6913 default:
6914 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6915 enmTrapType = TRPM_32BIT_HACK;
6916 break;
6917 }
6918
6919 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6920
6921 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6922 AssertRC(rc);
6923
6924 if (fErrorCodeValid)
6925 TRPMSetErrorCode(pVCpu, uErrorCode);
6926
6927 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6928 && uVector == X86_XCPT_PF)
6929 {
6930 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6931 }
6932 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6933 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6934 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6935 {
6936 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6937 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6938 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6939 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6940 }
6941 pVCpu->hm.s.Event.fPending = false;
6942}
6943
6944
6945/**
6946 * Does the necessary state syncing before returning to ring-3 for any reason
6947 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6948 *
6949 * @returns VBox status code.
6950 * @param pVM Pointer to the VM.
6951 * @param pVCpu Pointer to the VMCPU.
6952 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6953 * be out-of-sync. Make sure to update the required
6954 * fields before using them.
6955 * @param fSaveGuestState Whether to save the guest state or not.
6956 *
6957 * @remarks No-long-jmp zone!!!
6958 */
6959static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6960{
6961 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6962 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6963
6964 RTCPUID idCpu = RTMpCpuId();
6965 Log4Func(("HostCpuId=%u\n", idCpu));
6966
6967 /*
6968 * !!! IMPORTANT !!!
6969 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6970 */
6971
6972 /* Save the guest state if necessary. */
6973 if ( fSaveGuestState
6974 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6975 {
6976 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6977 AssertRCReturn(rc, rc);
6978 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6979 }
6980
6981 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6982 if (CPUMIsGuestFPUStateActive(pVCpu))
6983 {
6984 /* We shouldn't reload CR0 without saving it first. */
6985 if (!fSaveGuestState)
6986 {
6987 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6988 AssertRCReturn(rc, rc);
6989 }
6990 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6991 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6992 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6993 }
6994
6995 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6996#ifdef VBOX_STRICT
6997 if (CPUMIsHyperDebugStateActive(pVCpu))
6998 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6999#endif
7000 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7001 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7002 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7003 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7004
7005#if HC_ARCH_BITS == 64
7006 /* Restore host-state bits that VT-x only restores partially. */
7007 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7008 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7009 {
7010 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7011 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7012 }
7013 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7014#endif
7015
7016#if HC_ARCH_BITS == 64
7017 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7018 if ( pVM->hm.s.fAllow64BitGuests
7019 && pVCpu->hm.s.vmx.fLazyMsrs)
7020 {
7021 /* We shouldn't reload the guest MSRs without saving it first. */
7022 if (!fSaveGuestState)
7023 {
7024 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7025 AssertRCReturn(rc, rc);
7026 }
7027 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7028 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7029 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7030 }
7031#endif
7032
7033 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7034 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7035
7036 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7037 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7038 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7039 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7040 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7041 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7042 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7043 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7044
7045 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7046
7047 /** @todo This partially defeats the purpose of having preemption hooks.
7048 * The problem is, deregistering the hooks should be moved to a place that
7049 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7050 * context.
7051 */
7052 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7053 {
7054 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7055 AssertRCReturn(rc, rc);
7056
7057 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7058 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7059 }
7060 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7061 NOREF(idCpu);
7062
7063 return VINF_SUCCESS;
7064}
7065
7066
7067/**
7068 * Leaves the VT-x session.
7069 *
7070 * @returns VBox status code.
7071 * @param pVM Pointer to the VM.
7072 * @param pVCpu Pointer to the VMCPU.
7073 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7074 * out-of-sync. Make sure to update the required fields
7075 * before using them.
7076 *
7077 * @remarks No-long-jmp zone!!!
7078 */
7079DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7080{
7081 HM_DISABLE_PREEMPT_IF_NEEDED();
7082 HMVMX_ASSERT_CPU_SAFE();
7083 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7084 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7085
7086 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7087 and done this from the VMXR0ThreadCtxCallback(). */
7088 if (!pVCpu->hm.s.fLeaveDone)
7089 {
7090 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7091 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7092 pVCpu->hm.s.fLeaveDone = true;
7093 }
7094 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7095
7096 /*
7097 * !!! IMPORTANT !!!
7098 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7099 */
7100
7101 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7102 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7103 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7104 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7105 VMMR0ThreadCtxHooksDeregister(pVCpu);
7106
7107 /* Leave HM context. This takes care of local init (term). */
7108 int rc = HMR0LeaveCpu(pVCpu);
7109
7110 HM_RESTORE_PREEMPT_IF_NEEDED();
7111
7112 return rc;
7113}
7114
7115
7116/**
7117 * Does the necessary state syncing before doing a longjmp to ring-3.
7118 *
7119 * @returns VBox status code.
7120 * @param pVM Pointer to the VM.
7121 * @param pVCpu Pointer to the VMCPU.
7122 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7123 * out-of-sync. Make sure to update the required fields
7124 * before using them.
7125 *
7126 * @remarks No-long-jmp zone!!!
7127 */
7128DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7129{
7130 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7131}
7132
7133
7134/**
7135 * Take necessary actions before going back to ring-3.
7136 *
7137 * An action requires us to go back to ring-3. This function does the necessary
7138 * steps before we can safely return to ring-3. This is not the same as longjmps
7139 * to ring-3, this is voluntary and prepares the guest so it may continue
7140 * executing outside HM (recompiler/IEM).
7141 *
7142 * @returns VBox status code.
7143 * @param pVM Pointer to the VM.
7144 * @param pVCpu Pointer to the VMCPU.
7145 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7146 * out-of-sync. Make sure to update the required fields
7147 * before using them.
7148 * @param rcExit The reason for exiting to ring-3. Can be
7149 * VINF_VMM_UNKNOWN_RING3_CALL.
7150 */
7151static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7152{
7153 Assert(pVM);
7154 Assert(pVCpu);
7155 Assert(pMixedCtx);
7156 HMVMX_ASSERT_PREEMPT_SAFE();
7157
7158 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7159 {
7160 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7161 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7162 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7163 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7164 }
7165
7166 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7167 VMMRZCallRing3Disable(pVCpu);
7168 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7169
7170 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7171 if (pVCpu->hm.s.Event.fPending)
7172 {
7173 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7174 Assert(!pVCpu->hm.s.Event.fPending);
7175 }
7176
7177 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7178 and if we're injecting an event we should have a TRPM trap pending. */
7179 Assert(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu));
7180 Assert(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu));
7181
7182 /* Save guest state and restore host state bits. */
7183 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7184 AssertRCReturn(rc, rc);
7185 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7186 /* Thread-context hooks are unregistered at this point!!! */
7187
7188 /* Sync recompiler state. */
7189 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7190 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7191 | CPUM_CHANGED_LDTR
7192 | CPUM_CHANGED_GDTR
7193 | CPUM_CHANGED_IDTR
7194 | CPUM_CHANGED_TR
7195 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7196 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7197 if ( pVM->hm.s.fNestedPaging
7198 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7199 {
7200 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7201 }
7202
7203 Assert(!pVCpu->hm.s.fClearTrapFlag);
7204
7205 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7206 if (rcExit != VINF_EM_RAW_INTERRUPT)
7207 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7208
7209 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7210
7211 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7212 VMMRZCallRing3RemoveNotification(pVCpu);
7213 VMMRZCallRing3Enable(pVCpu);
7214
7215 return rc;
7216}
7217
7218
7219/**
7220 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7221 * longjump to ring-3 and possibly get preempted.
7222 *
7223 * @returns VBox status code.
7224 * @param pVCpu Pointer to the VMCPU.
7225 * @param enmOperation The operation causing the ring-3 longjump.
7226 * @param pvUser Opaque pointer to the guest-CPU context. The data
7227 * may be out-of-sync. Make sure to update the required
7228 * fields before using them.
7229 */
7230DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7231{
7232 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7233 {
7234 /*
7235 * !!! IMPORTANT !!!
7236 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7237 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7238 */
7239 VMMRZCallRing3RemoveNotification(pVCpu);
7240 VMMRZCallRing3Disable(pVCpu);
7241 HM_DISABLE_PREEMPT_IF_NEEDED();
7242
7243 PVM pVM = pVCpu->CTX_SUFF(pVM);
7244 if (CPUMIsGuestFPUStateActive(pVCpu))
7245 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7246
7247 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7248
7249#if HC_ARCH_BITS == 64
7250 /* Restore host-state bits that VT-x only restores partially. */
7251 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7252 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7253 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7254 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7255
7256 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7257 if ( pVM->hm.s.fAllow64BitGuests
7258 && pVCpu->hm.s.vmx.fLazyMsrs)
7259 {
7260 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7261 }
7262#endif
7263 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7264 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7265 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7266 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7267 {
7268 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7269 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7270 }
7271
7272 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7273 VMMR0ThreadCtxHooksDeregister(pVCpu);
7274
7275 HMR0LeaveCpu(pVCpu);
7276 HM_RESTORE_PREEMPT_IF_NEEDED();
7277 return VINF_SUCCESS;
7278 }
7279
7280 Assert(pVCpu);
7281 Assert(pvUser);
7282 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7283 HMVMX_ASSERT_PREEMPT_SAFE();
7284
7285 VMMRZCallRing3Disable(pVCpu);
7286 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7287
7288 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7289 enmOperation));
7290
7291 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7292 AssertRCReturn(rc, rc);
7293
7294 VMMRZCallRing3Enable(pVCpu);
7295 return VINF_SUCCESS;
7296}
7297
7298
7299/**
7300 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7301 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7302 *
7303 * @param pVCpu Pointer to the VMCPU.
7304 */
7305DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7306{
7307 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7308 {
7309 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7310 {
7311 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7312 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7313 AssertRC(rc);
7314 Log4(("Setup interrupt-window exiting\n"));
7315 }
7316 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7317}
7318
7319
7320/**
7321 * Clears the interrupt-window exiting control in the VMCS.
7322 *
7323 * @param pVCpu Pointer to the VMCPU.
7324 */
7325DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7326{
7327 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7328 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7329 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7330 AssertRC(rc);
7331 Log4(("Cleared interrupt-window exiting\n"));
7332}
7333
7334
7335/**
7336 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7337 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7338 *
7339 * @param pVCpu Pointer to the VMCPU.
7340 */
7341DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7342{
7343 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7344 {
7345 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7346 {
7347 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7348 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7349 AssertRC(rc);
7350 Log4(("Setup NMI-window exiting\n"));
7351 }
7352 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7353}
7354
7355
7356/**
7357 * Clears the NMI-window exiting control in the VMCS.
7358 *
7359 * @param pVCpu Pointer to the VMCPU.
7360 */
7361DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7362{
7363 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7364 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7365 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7366 AssertRC(rc);
7367 Log4(("Cleared NMI-window exiting\n"));
7368}
7369
7370
7371/**
7372 * Evaluates the event to be delivered to the guest and sets it as the pending
7373 * event.
7374 *
7375 * @param pVCpu Pointer to the VMCPU.
7376 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7377 * out-of-sync. Make sure to update the required fields
7378 * before using them.
7379 */
7380static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7381{
7382 Assert(!pVCpu->hm.s.Event.fPending);
7383
7384 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7385 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7386 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7387 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7388 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7389
7390 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7391 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7392 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7393 Assert(!TRPMHasTrap(pVCpu));
7394
7395 /*
7396 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7397 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7398 */
7399 /** @todo SMI. SMIs take priority over NMIs. */
7400 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7401 {
7402 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7403 if ( !fBlockNmi
7404 && !fBlockSti
7405 && !fBlockMovSS)
7406 {
7407 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7408 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7409 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7410
7411 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7412 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7413 }
7414 else
7415 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7416 }
7417 /*
7418 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7419 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7420 */
7421 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7422 && !pVCpu->hm.s.fSingleInstruction)
7423 {
7424 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7425 AssertRC(rc);
7426 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7427 if ( !fBlockInt
7428 && !fBlockSti
7429 && !fBlockMovSS)
7430 {
7431 uint8_t u8Interrupt;
7432 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7433 if (RT_SUCCESS(rc))
7434 {
7435 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7436 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7437 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7438
7439 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7440 }
7441 else
7442 {
7443 /** @todo Does this actually happen? If not turn it into an assertion. */
7444 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7445 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7446 }
7447 }
7448 else
7449 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7450 }
7451}
7452
7453
7454/**
7455 * Sets a pending-debug exception to be delivered to the guest if the guest is
7456 * single-stepping.
7457 *
7458 * @param pVCpu Pointer to the VMCPU.
7459 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7460 * out-of-sync. Make sure to update the required fields
7461 * before using them.
7462 */
7463DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7464{
7465 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7466 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7467 {
7468 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7469 AssertRC(rc);
7470 }
7471}
7472
7473
7474/**
7475 * Injects any pending events into the guest if the guest is in a state to
7476 * receive them.
7477 *
7478 * @returns VBox status code (informational status codes included).
7479 * @param pVCpu Pointer to the VMCPU.
7480 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7481 * out-of-sync. Make sure to update the required fields
7482 * before using them.
7483 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7484 * return VINF_EM_DBG_STEPPED if the event was
7485 * dispatched directly.
7486 */
7487static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7488{
7489 HMVMX_ASSERT_PREEMPT_SAFE();
7490 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7491
7492 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7493 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7494 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7495 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7496
7497 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7498 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7499 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7500 Assert(!TRPMHasTrap(pVCpu));
7501
7502 int rc = VINF_SUCCESS;
7503 if (pVCpu->hm.s.Event.fPending)
7504 {
7505 /*
7506 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7507 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7508 * ended up enabling interrupts outside VT-x.
7509 */
7510 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7511 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7512 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7513 {
7514 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7515 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7516 }
7517
7518#ifdef VBOX_STRICT
7519 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7520 {
7521 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7522 Assert(!fBlockInt);
7523 Assert(!fBlockSti);
7524 Assert(!fBlockMovSS);
7525 }
7526 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7527 {
7528 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7529 Assert(!fBlockSti);
7530 Assert(!fBlockMovSS);
7531 Assert(!fBlockNmi);
7532 }
7533#endif
7534 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7535 (uint8_t)uIntType));
7536 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7537 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7538 AssertRCReturn(rc, rc);
7539
7540 /* Update the interruptibility-state as it could have been changed by
7541 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7542 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7543 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7544
7545#ifdef VBOX_WITH_STATISTICS
7546 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7547 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7548 else
7549 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7550#endif
7551 }
7552
7553 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7554 if ( fBlockSti
7555 || fBlockMovSS)
7556 {
7557 if ( !pVCpu->hm.s.fSingleInstruction
7558 && !DBGFIsStepping(pVCpu))
7559 {
7560 /*
7561 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7562 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7563 * See Intel spec. 27.3.4 "Saving Non-Register State".
7564 */
7565 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7566 AssertRCReturn(rc2, rc2);
7567 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7568 }
7569 else if (pMixedCtx->eflags.Bits.u1TF)
7570 {
7571 /*
7572 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7573 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7574 */
7575 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7576 uIntrState = 0;
7577 }
7578 }
7579
7580 /*
7581 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7582 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7583 */
7584 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7585 AssertRC(rc2);
7586
7587 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7588 NOREF(fBlockMovSS); NOREF(fBlockSti);
7589 return rc;
7590}
7591
7592
7593/**
7594 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7595 *
7596 * @param pVCpu Pointer to the VMCPU.
7597 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7598 * out-of-sync. Make sure to update the required fields
7599 * before using them.
7600 */
7601DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7602{
7603 NOREF(pMixedCtx);
7604 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7605 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7606}
7607
7608
7609/**
7610 * Injects a double-fault (#DF) exception into the VM.
7611 *
7612 * @returns VBox status code (informational status code included).
7613 * @param pVCpu Pointer to the VMCPU.
7614 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7615 * out-of-sync. Make sure to update the required fields
7616 * before using them.
7617 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7618 * and should return VINF_EM_DBG_STEPPED if the event
7619 * is injected directly (register modified by us, not
7620 * by hardware on VM-entry).
7621 * @param puIntrState Pointer to the current guest interruptibility-state.
7622 * This interruptibility-state will be updated if
7623 * necessary. This cannot not be NULL.
7624 */
7625DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7626{
7627 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7628 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7629 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7630 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7631 fStepping, puIntrState);
7632}
7633
7634
7635/**
7636 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7637 *
7638 * @param pVCpu Pointer to the VMCPU.
7639 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7640 * out-of-sync. Make sure to update the required fields
7641 * before using them.
7642 */
7643DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7644{
7645 NOREF(pMixedCtx);
7646 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7647 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7648 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7649}
7650
7651
7652/**
7653 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7654 *
7655 * @param pVCpu Pointer to the VMCPU.
7656 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7657 * out-of-sync. Make sure to update the required fields
7658 * before using them.
7659 * @param cbInstr The value of RIP that is to be pushed on the guest
7660 * stack.
7661 */
7662DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7663{
7664 NOREF(pMixedCtx);
7665 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7666 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7667 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7668}
7669
7670
7671/**
7672 * Injects a general-protection (#GP) fault into the VM.
7673 *
7674 * @returns VBox status code (informational status code included).
7675 * @param pVCpu Pointer to the VMCPU.
7676 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7677 * out-of-sync. Make sure to update the required fields
7678 * before using them.
7679 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7680 * mode, i.e. in real-mode it's not valid).
7681 * @param u32ErrorCode The error code associated with the #GP.
7682 * @param fStepping Whether we're running in
7683 * hmR0VmxRunGuestCodeStep() and should return
7684 * VINF_EM_DBG_STEPPED if the event is injected
7685 * directly (register modified by us, not by
7686 * hardware on VM-entry).
7687 * @param puIntrState Pointer to the current guest interruptibility-state.
7688 * This interruptibility-state will be updated if
7689 * necessary. This cannot not be NULL.
7690 */
7691DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7692 bool fStepping, uint32_t *puIntrState)
7693{
7694 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7695 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7696 if (fErrorCodeValid)
7697 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7698 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7699 fStepping, puIntrState);
7700}
7701
7702
7703/**
7704 * Sets a general-protection (#GP) exception as pending-for-injection into the
7705 * VM.
7706 *
7707 * @param pVCpu Pointer to the VMCPU.
7708 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7709 * out-of-sync. Make sure to update the required fields
7710 * before using them.
7711 * @param u32ErrorCode The error code associated with the #GP.
7712 */
7713DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7714{
7715 NOREF(pMixedCtx);
7716 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7717 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7718 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7719 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7720}
7721
7722
7723/**
7724 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7725 *
7726 * @param pVCpu Pointer to the VMCPU.
7727 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7728 * out-of-sync. Make sure to update the required fields
7729 * before using them.
7730 * @param uVector The software interrupt vector number.
7731 * @param cbInstr The value of RIP that is to be pushed on the guest
7732 * stack.
7733 */
7734DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7735{
7736 NOREF(pMixedCtx);
7737 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7738 if ( uVector == X86_XCPT_BP
7739 || uVector == X86_XCPT_OF)
7740 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7741 else
7742 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7743 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7744}
7745
7746
7747/**
7748 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7749 * stack.
7750 *
7751 * @returns VBox status code (information status code included).
7752 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7753 * @param pVM Pointer to the VM.
7754 * @param pMixedCtx Pointer to the guest-CPU context.
7755 * @param uValue The value to push to the guest stack.
7756 */
7757DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7758{
7759 /*
7760 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7761 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7762 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7763 */
7764 if (pMixedCtx->sp == 1)
7765 return VINF_EM_RESET;
7766 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7767 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7768 AssertRCReturn(rc, rc);
7769 return rc;
7770}
7771
7772
7773/**
7774 * Injects an event into the guest upon VM-entry by updating the relevant fields
7775 * in the VM-entry area in the VMCS.
7776 *
7777 * @returns VBox status code (informational error codes included).
7778 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7779 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7780 *
7781 * @param pVCpu Pointer to the VMCPU.
7782 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7783 * be out-of-sync. Make sure to update the required
7784 * fields before using them.
7785 * @param u64IntInfo The VM-entry interruption-information field.
7786 * @param cbInstr The VM-entry instruction length in bytes (for
7787 * software interrupts, exceptions and privileged
7788 * software exceptions).
7789 * @param u32ErrCode The VM-entry exception error code.
7790 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7791 * @param puIntrState Pointer to the current guest interruptibility-state.
7792 * This interruptibility-state will be updated if
7793 * necessary. This cannot not be NULL.
7794 * @param fStepping Whether we're running in
7795 * hmR0VmxRunGuestCodeStep() and should return
7796 * VINF_EM_DBG_STEPPED if the event is injected
7797 * directly (register modified by us, not by
7798 * hardware on VM-entry).
7799 *
7800 * @remarks Requires CR0!
7801 * @remarks No-long-jump zone!!!
7802 */
7803static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7804 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7805{
7806 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7807 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7808 Assert(puIntrState);
7809 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7810
7811 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7812 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7813
7814#ifdef VBOX_STRICT
7815 /* Validate the error-code-valid bit for hardware exceptions. */
7816 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7817 {
7818 switch (uVector)
7819 {
7820 case X86_XCPT_PF:
7821 case X86_XCPT_DF:
7822 case X86_XCPT_TS:
7823 case X86_XCPT_NP:
7824 case X86_XCPT_SS:
7825 case X86_XCPT_GP:
7826 case X86_XCPT_AC:
7827 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7828 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7829 /* fallthru */
7830 default:
7831 break;
7832 }
7833 }
7834#endif
7835
7836 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7837 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7838 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7839
7840 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7841
7842 /* We require CR0 to check if the guest is in real-mode. */
7843 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7844 AssertRCReturn(rc, rc);
7845
7846 /*
7847 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7848 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7849 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7850 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7851 */
7852 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7853 {
7854 PVM pVM = pVCpu->CTX_SUFF(pVM);
7855 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7856 {
7857 Assert(PDMVmmDevHeapIsEnabled(pVM));
7858 Assert(pVM->hm.s.vmx.pRealModeTSS);
7859
7860 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7861 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7862 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7863 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7864 AssertRCReturn(rc, rc);
7865 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7866
7867 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7868 size_t const cbIdtEntry = sizeof(X86IDTR16);
7869 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7870 {
7871 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7872 if (uVector == X86_XCPT_DF)
7873 return VINF_EM_RESET;
7874
7875 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7876 if (uVector == X86_XCPT_GP)
7877 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7878
7879 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7880 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7881 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7882 fStepping, puIntrState);
7883 }
7884
7885 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7886 uint16_t uGuestIp = pMixedCtx->ip;
7887 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7888 {
7889 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7890 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7891 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7892 }
7893 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7894 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7895
7896 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7897 X86IDTR16 IdtEntry;
7898 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7899 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7900 AssertRCReturn(rc, rc);
7901
7902 /* Construct the stack frame for the interrupt/exception handler. */
7903 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7904 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7905 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7906 AssertRCReturn(rc, rc);
7907
7908 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7909 if (rc == VINF_SUCCESS)
7910 {
7911 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7912 pMixedCtx->rip = IdtEntry.offSel;
7913 pMixedCtx->cs.Sel = IdtEntry.uSel;
7914 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7915 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7916 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7917 && uVector == X86_XCPT_PF)
7918 pMixedCtx->cr2 = GCPtrFaultAddress;
7919
7920 /* If any other guest-state bits are changed here, make sure to update
7921 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7922 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7923 | HM_CHANGED_GUEST_RIP
7924 | HM_CHANGED_GUEST_RFLAGS
7925 | HM_CHANGED_GUEST_RSP);
7926
7927 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7928 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7929 {
7930 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7931 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7932 Log4(("Clearing inhibition due to STI.\n"));
7933 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7934 }
7935 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7936 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7937
7938 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7939 it, if we are returning to ring-3 before executing guest code. */
7940 pVCpu->hm.s.Event.fPending = false;
7941
7942 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7943 if (fStepping)
7944 rc = VINF_EM_DBG_STEPPED;
7945 }
7946 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7947 return rc;
7948 }
7949
7950 /*
7951 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7952 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7953 */
7954 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7955 }
7956
7957 /* Validate. */
7958 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7959 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7960 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7961
7962 /* Inject. */
7963 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7964 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7965 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7966 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7967
7968 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7969 && uVector == X86_XCPT_PF)
7970 pMixedCtx->cr2 = GCPtrFaultAddress;
7971
7972 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7973 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7974
7975 AssertRCReturn(rc, rc);
7976 return rc;
7977}
7978
7979
7980/**
7981 * Clears the interrupt-window exiting control in the VMCS and if necessary
7982 * clears the current event in the VMCS as well.
7983 *
7984 * @returns VBox status code.
7985 * @param pVCpu Pointer to the VMCPU.
7986 *
7987 * @remarks Use this function only to clear events that have not yet been
7988 * delivered to the guest but are injected in the VMCS!
7989 * @remarks No-long-jump zone!!!
7990 */
7991static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7992{
7993 int rc;
7994 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7995
7996 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7997 {
7998 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7999 Assert(!pVCpu->hm.s.Event.fPending);
8000 }
8001
8002 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8003 {
8004 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8005 Assert(!pVCpu->hm.s.Event.fPending);
8006 }
8007
8008 if (!pVCpu->hm.s.Event.fPending)
8009 return;
8010
8011#ifdef VBOX_STRICT
8012 uint32_t u32EntryInfo;
8013 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8014 AssertRC(rc);
8015 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8016#endif
8017
8018 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8019 AssertRC(rc);
8020
8021 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8022 AssertRC(rc);
8023
8024 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8025 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8026}
8027
8028
8029/**
8030 * Enters the VT-x session.
8031 *
8032 * @returns VBox status code.
8033 * @param pVM Pointer to the VM.
8034 * @param pVCpu Pointer to the VMCPU.
8035 * @param pCpu Pointer to the CPU info struct.
8036 */
8037VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8038{
8039 AssertPtr(pVM);
8040 AssertPtr(pVCpu);
8041 Assert(pVM->hm.s.vmx.fSupported);
8042 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8043 NOREF(pCpu); NOREF(pVM);
8044
8045 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8046 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8047
8048#ifdef VBOX_STRICT
8049 /* Make sure we're in VMX root mode. */
8050 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8051 if (!(u32HostCR4 & X86_CR4_VMXE))
8052 {
8053 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8054 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8055 }
8056#endif
8057
8058 /*
8059 * Load the VCPU's VMCS as the current (and active) one.
8060 */
8061 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8062 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8063 if (RT_FAILURE(rc))
8064 return rc;
8065
8066 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8067 pVCpu->hm.s.fLeaveDone = false;
8068 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8069
8070 return VINF_SUCCESS;
8071}
8072
8073
8074/**
8075 * The thread-context callback (only on platforms which support it).
8076 *
8077 * @param enmEvent The thread-context event.
8078 * @param pVCpu Pointer to the VMCPU.
8079 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8080 * @thread EMT(pVCpu)
8081 */
8082VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8083{
8084 NOREF(fGlobalInit);
8085
8086 switch (enmEvent)
8087 {
8088 case RTTHREADCTXEVENT_PREEMPTING:
8089 {
8090 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8091 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8092 VMCPU_ASSERT_EMT(pVCpu);
8093
8094 PVM pVM = pVCpu->CTX_SUFF(pVM);
8095 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8096
8097 /* No longjmps (logger flushes, locks) in this fragile context. */
8098 VMMRZCallRing3Disable(pVCpu);
8099 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8100
8101 /*
8102 * Restore host-state (FPU, debug etc.)
8103 */
8104 if (!pVCpu->hm.s.fLeaveDone)
8105 {
8106 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8107 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8108 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8109 pVCpu->hm.s.fLeaveDone = true;
8110 }
8111
8112 /* Leave HM context, takes care of local init (term). */
8113 int rc = HMR0LeaveCpu(pVCpu);
8114 AssertRC(rc); NOREF(rc);
8115
8116 /* Restore longjmp state. */
8117 VMMRZCallRing3Enable(pVCpu);
8118 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8119 break;
8120 }
8121
8122 case RTTHREADCTXEVENT_RESUMED:
8123 {
8124 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8125 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8126 VMCPU_ASSERT_EMT(pVCpu);
8127
8128 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8129 VMMRZCallRing3Disable(pVCpu);
8130 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8131
8132 /* Initialize the bare minimum state required for HM. This takes care of
8133 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8134 int rc = HMR0EnterCpu(pVCpu);
8135 AssertRC(rc);
8136 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8137
8138 /* Load the active VMCS as the current one. */
8139 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8140 {
8141 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8142 AssertRC(rc); NOREF(rc);
8143 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8144 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8145 }
8146 pVCpu->hm.s.fLeaveDone = false;
8147
8148 /* Restore longjmp state. */
8149 VMMRZCallRing3Enable(pVCpu);
8150 break;
8151 }
8152
8153 default:
8154 break;
8155 }
8156}
8157
8158
8159/**
8160 * Saves the host state in the VMCS host-state.
8161 * Sets up the VM-exit MSR-load area.
8162 *
8163 * The CPU state will be loaded from these fields on every successful VM-exit.
8164 *
8165 * @returns VBox status code.
8166 * @param pVM Pointer to the VM.
8167 * @param pVCpu Pointer to the VMCPU.
8168 *
8169 * @remarks No-long-jump zone!!!
8170 */
8171static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8172{
8173 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8174
8175 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8176 return VINF_SUCCESS;
8177
8178 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8179 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8180
8181 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8182 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8183
8184 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8185 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8186
8187 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8188 return rc;
8189}
8190
8191
8192/**
8193 * Saves the host state in the VMCS host-state.
8194 *
8195 * @returns VBox status code.
8196 * @param pVM Pointer to the VM.
8197 * @param pVCpu Pointer to the VMCPU.
8198 *
8199 * @remarks No-long-jump zone!!!
8200 */
8201VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8202{
8203 AssertPtr(pVM);
8204 AssertPtr(pVCpu);
8205
8206 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8207
8208 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8209 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8210 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8211 return hmR0VmxSaveHostState(pVM, pVCpu);
8212}
8213
8214
8215/**
8216 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8217 * loaded from these fields on every successful VM-entry.
8218 *
8219 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8220 * Sets up the VM-entry controls.
8221 * Sets up the appropriate VMX non-root function to execute guest code based on
8222 * the guest CPU mode.
8223 *
8224 * @returns VBox status code.
8225 * @param pVM Pointer to the VM.
8226 * @param pVCpu Pointer to the VMCPU.
8227 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8228 * out-of-sync. Make sure to update the required fields
8229 * before using them.
8230 *
8231 * @remarks No-long-jump zone!!!
8232 */
8233static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8234{
8235 AssertPtr(pVM);
8236 AssertPtr(pVCpu);
8237 AssertPtr(pMixedCtx);
8238 HMVMX_ASSERT_PREEMPT_SAFE();
8239
8240 VMMRZCallRing3Disable(pVCpu);
8241 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8242
8243 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8244
8245 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8246
8247 /* Determine real-on-v86 mode. */
8248 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8249 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8250 && CPUMIsGuestInRealModeEx(pMixedCtx))
8251 {
8252 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8253 }
8254
8255 /*
8256 * Load the guest-state into the VMCS.
8257 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8258 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8259 */
8260 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8261 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8262
8263 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8264 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8265 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8266
8267 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8268 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8269 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8270
8271 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8272 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8273
8274 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8275 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8276
8277 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8278 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8279 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8280
8281 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8282 determine we don't have to swap EFER after all. */
8283 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8284 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8285
8286 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8287 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8288
8289 /*
8290 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8291 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8292 */
8293 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8294 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8295
8296 /* Clear any unused and reserved bits. */
8297 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8298
8299 VMMRZCallRing3Enable(pVCpu);
8300
8301 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8302 return rc;
8303}
8304
8305
8306/**
8307 * Loads the state shared between the host and guest into the VMCS.
8308 *
8309 * @param pVM Pointer to the VM.
8310 * @param pVCpu Pointer to the VMCPU.
8311 * @param pCtx Pointer to the guest-CPU context.
8312 *
8313 * @remarks No-long-jump zone!!!
8314 */
8315static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8316{
8317 NOREF(pVM);
8318
8319 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8320 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8321
8322 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8323 {
8324 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8325 AssertRC(rc);
8326 }
8327
8328 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8329 {
8330 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8331 AssertRC(rc);
8332
8333 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8334 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8335 {
8336 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8337 AssertRC(rc);
8338 }
8339 }
8340
8341 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8342 {
8343#if HC_ARCH_BITS == 64
8344 if (pVM->hm.s.fAllow64BitGuests)
8345 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8346#endif
8347 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8348 }
8349
8350 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8351 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8352}
8353
8354
8355/**
8356 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8357 *
8358 * @param pVM Pointer to the VM.
8359 * @param pVCpu Pointer to the VMCPU.
8360 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8361 * out-of-sync. Make sure to update the required fields
8362 * before using them.
8363 */
8364DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8365{
8366 HMVMX_ASSERT_PREEMPT_SAFE();
8367
8368 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8369#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8370 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8371#endif
8372
8373 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8374 {
8375 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8376 AssertRC(rc);
8377 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8378 }
8379 else if (HMCPU_CF_VALUE(pVCpu))
8380 {
8381 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8382 AssertRC(rc);
8383 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8384 }
8385
8386 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8387 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8388 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8389 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8390}
8391
8392
8393/**
8394 * Does the preparations before executing guest code in VT-x.
8395 *
8396 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8397 * recompiler/IEM. We must be cautious what we do here regarding committing
8398 * guest-state information into the VMCS assuming we assuredly execute the
8399 * guest in VT-x mode.
8400 *
8401 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8402 * the common-state (TRPM/forceflags), we must undo those changes so that the
8403 * recompiler/IEM can (and should) use them when it resumes guest execution.
8404 * Otherwise such operations must be done when we can no longer exit to ring-3.
8405 *
8406 * @returns Strict VBox status code.
8407 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8408 * have been disabled.
8409 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8410 * double-fault into the guest.
8411 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8412 * dispatched directly.
8413 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8414 *
8415 * @param pVM Pointer to the VM.
8416 * @param pVCpu Pointer to the VMCPU.
8417 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8418 * out-of-sync. Make sure to update the required fields
8419 * before using them.
8420 * @param pVmxTransient Pointer to the VMX transient structure.
8421 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8422 * us ignore some of the reasons for returning to
8423 * ring-3, and return VINF_EM_DBG_STEPPED if event
8424 * dispatching took place.
8425 */
8426static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8427{
8428 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8429
8430#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8431 PGMRZDynMapFlushAutoSet(pVCpu);
8432#endif
8433
8434 /* Check force flag actions that might require us to go back to ring-3. */
8435 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8436 if (rc != VINF_SUCCESS)
8437 return rc;
8438
8439#ifndef IEM_VERIFICATION_MODE_FULL
8440 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8441 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8442 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8443 {
8444 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8445 RTGCPHYS GCPhysApicBase;
8446 GCPhysApicBase = pMixedCtx->msrApicBase;
8447 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8448
8449 /* Unalias any existing mapping. */
8450 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8451 AssertRCReturn(rc, rc);
8452
8453 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8454 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8455 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8456 AssertRCReturn(rc, rc);
8457
8458 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8459 }
8460#endif /* !IEM_VERIFICATION_MODE_FULL */
8461
8462 if (TRPMHasTrap(pVCpu))
8463 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8464 else if (!pVCpu->hm.s.Event.fPending)
8465 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8466
8467 /*
8468 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8469 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8470 */
8471 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8472 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8473 {
8474 Assert(rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
8475 return rc;
8476 }
8477
8478 /*
8479 * Load the guest state bits, we can handle longjmps/getting preempted here.
8480 *
8481 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8482 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8483 * Hence, this needs to be done -after- injection of events.
8484 */
8485 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8486
8487 /*
8488 * No longjmps to ring-3 from this point on!!!
8489 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8490 * This also disables flushing of the R0-logger instance (if any).
8491 */
8492 VMMRZCallRing3Disable(pVCpu);
8493
8494 /*
8495 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8496 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8497 *
8498 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8499 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8500 *
8501 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8502 * executing guest code.
8503 */
8504 pVmxTransient->uEflags = ASMIntDisableFlags();
8505 if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8506 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8507 && ( !fStepping /* Optimized for the non-stepping case, of course. */
8508 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8509 {
8510 hmR0VmxClearEventVmcs(pVCpu);
8511 ASMSetFlags(pVmxTransient->uEflags);
8512 VMMRZCallRing3Enable(pVCpu);
8513 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8514 return VINF_EM_RAW_TO_R3;
8515 }
8516
8517 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8518 {
8519 hmR0VmxClearEventVmcs(pVCpu);
8520 ASMSetFlags(pVmxTransient->uEflags);
8521 VMMRZCallRing3Enable(pVCpu);
8522 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8523 return VINF_EM_RAW_INTERRUPT;
8524 }
8525
8526 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8527 pVCpu->hm.s.Event.fPending = false;
8528
8529 return VINF_SUCCESS;
8530}
8531
8532
8533/**
8534 * Prepares to run guest code in VT-x and we've committed to doing so. This
8535 * means there is no backing out to ring-3 or anywhere else at this
8536 * point.
8537 *
8538 * @param pVM Pointer to the VM.
8539 * @param pVCpu Pointer to the VMCPU.
8540 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8541 * out-of-sync. Make sure to update the required fields
8542 * before using them.
8543 * @param pVmxTransient Pointer to the VMX transient structure.
8544 *
8545 * @remarks Called with preemption disabled.
8546 * @remarks No-long-jump zone!!!
8547 */
8548static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8549{
8550 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8551 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8552 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8553
8554 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8555 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8556
8557#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8558 if (!CPUMIsGuestFPUStateActive(pVCpu))
8559 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8560 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8561#endif
8562
8563 if ( pVCpu->hm.s.fUseGuestFpu
8564 && !CPUMIsGuestFPUStateActive(pVCpu))
8565 {
8566 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8567 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8568 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8569 }
8570
8571 /*
8572 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8573 */
8574 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8575 && pVCpu->hm.s.vmx.cMsrs > 0)
8576 {
8577 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8578 }
8579
8580 /*
8581 * Load the host state bits as we may've been preempted (only happens when
8582 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8583 */
8584 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8585 {
8586 /* This ASSUMES that pfnStartVM has been set up already. */
8587 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8588 AssertRC(rc);
8589 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8590 }
8591 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8592
8593 /*
8594 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8595 */
8596 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8597 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8598 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8599
8600 /* Store status of the shared guest-host state at the time of VM-entry. */
8601#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8602 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8603 {
8604 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8605 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8606 }
8607 else
8608#endif
8609 {
8610 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8611 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8612 }
8613 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8614
8615 /*
8616 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8617 */
8618 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8619 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8620
8621 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8622 RTCPUID idCurrentCpu = pCpu->idCpu;
8623 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8624 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8625 {
8626 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8627 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8628 }
8629
8630 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8631 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8632 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8633 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8634
8635 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8636
8637 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8638 to start executing. */
8639
8640 /*
8641 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8642 */
8643 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8644 {
8645 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8646 {
8647 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8648 AssertRC(rc2);
8649 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8650 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8651 true /* fUpdateHostMsr */);
8652 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8653 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8654 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8655 }
8656 else
8657 {
8658 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8659 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8660 }
8661 }
8662
8663#ifdef VBOX_STRICT
8664 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8665 hmR0VmxCheckHostEferMsr(pVCpu);
8666 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8667#endif
8668#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8669 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8670 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8671 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8672#endif
8673}
8674
8675
8676/**
8677 * Performs some essential restoration of state after running guest code in
8678 * VT-x.
8679 *
8680 * @param pVM Pointer to the VM.
8681 * @param pVCpu Pointer to the VMCPU.
8682 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8683 * out-of-sync. Make sure to update the required fields
8684 * before using them.
8685 * @param pVmxTransient Pointer to the VMX transient structure.
8686 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8687 *
8688 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8689 *
8690 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8691 * unconditionally when it is safe to do so.
8692 */
8693static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8694{
8695 NOREF(pVM);
8696
8697 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8698
8699 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8700 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8701 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8702 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8703 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8704 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8705
8706 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8707 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset); /** @todo use SUPReadTSC() eventually. */
8708
8709 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8710 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8711 Assert(!(ASMGetFlags() & X86_EFL_IF));
8712 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8713
8714#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8715 if (CPUMIsGuestFPUStateActive(pVCpu))
8716 {
8717 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8718 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8719 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8720 }
8721#endif
8722
8723#if HC_ARCH_BITS == 64
8724 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8725#endif
8726 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8727#ifdef VBOX_STRICT
8728 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8729#endif
8730 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8731 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8732
8733 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8734 uint32_t uExitReason;
8735 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8736 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8737 AssertRC(rc);
8738 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8739 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8740
8741 /* Update the VM-exit history array. */
8742 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8743
8744 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8745 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8746 {
8747 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8748 pVmxTransient->fVMEntryFailed));
8749 return;
8750 }
8751
8752 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8753 {
8754 /** @todo We can optimize this by only syncing with our force-flags when
8755 * really needed and keeping the VMCS state as it is for most
8756 * VM-exits. */
8757 /* Update the guest interruptibility-state from the VMCS. */
8758 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8759
8760#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8761 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8762 AssertRC(rc);
8763#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8764 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8765 AssertRC(rc);
8766#endif
8767
8768 /*
8769 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8770 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8771 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8772 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8773 */
8774 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8775 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8776 {
8777 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8778 AssertRC(rc);
8779 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8780 }
8781 }
8782}
8783
8784
8785/**
8786 * Runs the guest code using VT-x the normal way.
8787 *
8788 * @returns VBox status code.
8789 * @param pVM Pointer to the VM.
8790 * @param pVCpu Pointer to the VMCPU.
8791 * @param pCtx Pointer to the guest-CPU context.
8792 *
8793 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8794 */
8795static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8796{
8797 VMXTRANSIENT VmxTransient;
8798 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8799 int rc = VERR_INTERNAL_ERROR_5;
8800 uint32_t cLoops = 0;
8801
8802 for (;; cLoops++)
8803 {
8804 Assert(!HMR0SuspendPending());
8805 HMVMX_ASSERT_CPU_SAFE();
8806
8807 /* Preparatory work for running guest code, this may force us to return
8808 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8809 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8810 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8811 if (rc != VINF_SUCCESS)
8812 break;
8813
8814 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8815 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8816 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8817
8818 /* Restore any residual host-state and save any bits shared between host
8819 and guest into the guest-CPU state. Re-enables interrupts! */
8820 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8821
8822 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8823 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8824 {
8825 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8826 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8827 return rc;
8828 }
8829
8830 /* Profile the VM-exit. */
8831 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8832 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8833 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8834 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8835 HMVMX_START_EXIT_DISPATCH_PROF();
8836
8837 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8838 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8839 {
8840 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8841 hmR0VmxSaveGuestState(pVCpu, pCtx);
8842 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8843 }
8844
8845 /* Handle the VM-exit. */
8846#ifdef HMVMX_USE_FUNCTION_TABLE
8847 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8848#else
8849 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8850#endif
8851 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8852 if (rc != VINF_SUCCESS)
8853 break;
8854 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8855 {
8856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8857 rc = VINF_EM_RAW_INTERRUPT;
8858 break;
8859 }
8860 }
8861
8862 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8863 return rc;
8864}
8865
8866
8867/**
8868 * Single steps guest code using VT-x.
8869 *
8870 * @returns VBox status code.
8871 * @param pVM Pointer to the VM.
8872 * @param pVCpu Pointer to the VMCPU.
8873 * @param pCtx Pointer to the guest-CPU context.
8874 *
8875 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8876 */
8877static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8878{
8879 VMXTRANSIENT VmxTransient;
8880 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8881 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8882 uint32_t cLoops = 0;
8883 uint16_t uCsStart = pCtx->cs.Sel;
8884 uint64_t uRipStart = pCtx->rip;
8885
8886 for (;; cLoops++)
8887 {
8888 Assert(!HMR0SuspendPending());
8889 HMVMX_ASSERT_CPU_SAFE();
8890
8891 /* Preparatory work for running guest code, this may force us to return
8892 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8893 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8894 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8895 if (rcStrict != VINF_SUCCESS)
8896 break;
8897
8898 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8899 rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8900 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8901
8902 /* Restore any residual host-state and save any bits shared between host
8903 and guest into the guest-CPU state. Re-enables interrupts! */
8904 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8905
8906 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8907 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
8908 {
8909 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8910 hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
8911 return VBOXSTRICTRC_TODO(rcStrict);
8912 }
8913
8914 /* Profile the VM-exit. */
8915 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8917 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8918 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8919 HMVMX_START_EXIT_DISPATCH_PROF();
8920
8921 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8922 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8923 {
8924 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8925 hmR0VmxSaveGuestState(pVCpu, pCtx);
8926 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8927 }
8928
8929 /* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8930 rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8931 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8932 if (rcStrict != VINF_SUCCESS)
8933 break;
8934 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8935 {
8936 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8937 rcStrict = VINF_EM_RAW_INTERRUPT;
8938 break;
8939 }
8940
8941 /*
8942 * Did the RIP change, if so, consider it a single step.
8943 * Otherwise, make sure one of the TFs gets set.
8944 */
8945 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8946 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8947 AssertRCReturn(rc2, rc2);
8948 if ( pCtx->rip != uRipStart
8949 || pCtx->cs.Sel != uCsStart)
8950 {
8951 rcStrict = VINF_EM_DBG_STEPPED;
8952 break;
8953 }
8954 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8955 }
8956
8957 /*
8958 * Clear the X86_EFL_TF if necessary.
8959 */
8960 if (pVCpu->hm.s.fClearTrapFlag)
8961 {
8962 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8963 AssertRCReturn(rc2, rc2);
8964 pVCpu->hm.s.fClearTrapFlag = false;
8965 pCtx->eflags.Bits.u1TF = 0;
8966 }
8967 /** @todo there seems to be issues with the resume flag when the monitor trap
8968 * flag is pending without being used. Seen early in bios init when
8969 * accessing APIC page in protected mode. */
8970
8971 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8972 return VBOXSTRICTRC_TODO(rcStrict);
8973}
8974
8975
8976/**
8977 * Runs the guest code using VT-x.
8978 *
8979 * @returns VBox status code.
8980 * @param pVM Pointer to the VM.
8981 * @param pVCpu Pointer to the VMCPU.
8982 * @param pCtx Pointer to the guest-CPU context.
8983 */
8984VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8985{
8986 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8987 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8988 HMVMX_ASSERT_PREEMPT_SAFE();
8989
8990 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8991
8992 int rc;
8993 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8994 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8995 else
8996 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8997
8998 if (rc == VERR_EM_INTERPRETER)
8999 rc = VINF_EM_RAW_EMULATE_INSTR;
9000 else if (rc == VINF_EM_RESET)
9001 rc = VINF_EM_TRIPLE_FAULT;
9002
9003 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
9004 if (RT_FAILURE(rc2))
9005 {
9006 pVCpu->hm.s.u32HMError = rc;
9007 rc = rc2;
9008 }
9009 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
9010 return rc;
9011}
9012
9013
9014#ifndef HMVMX_USE_FUNCTION_TABLE
9015DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9016{
9017#ifdef DEBUG_ramshankar
9018# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9019# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9020#endif
9021 int rc;
9022 switch (rcReason)
9023 {
9024 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9025 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9026 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9027 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9028 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9029 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9030 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9031 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9032 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9033 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9034 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9035 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9036 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9037 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9038 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9039 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9040 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9041 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9042 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9043 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9044 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9045 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9046 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9047 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9048 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9049 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9050 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9051 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9052 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9053 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9054 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9055 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9056 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9057 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9058
9059 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9060 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9061 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9062 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9063 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9064 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9065 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9066 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9067 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9068
9069 case VMX_EXIT_VMCLEAR:
9070 case VMX_EXIT_VMLAUNCH:
9071 case VMX_EXIT_VMPTRLD:
9072 case VMX_EXIT_VMPTRST:
9073 case VMX_EXIT_VMREAD:
9074 case VMX_EXIT_VMRESUME:
9075 case VMX_EXIT_VMWRITE:
9076 case VMX_EXIT_VMXOFF:
9077 case VMX_EXIT_VMXON:
9078 case VMX_EXIT_INVEPT:
9079 case VMX_EXIT_INVVPID:
9080 case VMX_EXIT_VMFUNC:
9081 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9082 break;
9083 default:
9084 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9085 break;
9086 }
9087 return rc;
9088}
9089#endif /* !HMVMX_USE_FUNCTION_TABLE */
9090
9091
9092/**
9093 * Single-stepping VM-exit filtering.
9094 *
9095 * This is preprocessing the exits and deciding whether we've gotten far enough
9096 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9097 * performed.
9098 *
9099 * @returns Strict VBox status code.
9100 * @param pVCpu The virtual CPU of the calling EMT.
9101 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9102 * out-of-sync. Make sure to update the required
9103 * fields before using them.
9104 * @param pVmxTransient Pointer to the VMX-transient structure.
9105 * @param uExitReason The VM-exit reason.
9106 */
9107DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9108 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9109{
9110 switch (uExitReason)
9111 {
9112 case VMX_EXIT_XCPT_OR_NMI:
9113 {
9114 /* Check for host NMI. */
9115 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9116 AssertRCReturn(rc2, rc2);
9117 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9118 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9119 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9120 /* fall thru */
9121 }
9122
9123 case VMX_EXIT_EPT_MISCONFIG:
9124 case VMX_EXIT_TRIPLE_FAULT:
9125 case VMX_EXIT_APIC_ACCESS:
9126 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9127 case VMX_EXIT_TASK_SWITCH:
9128
9129 /* Instruction specific VM-exits: */
9130 case VMX_EXIT_IO_INSTR:
9131 case VMX_EXIT_CPUID:
9132 case VMX_EXIT_RDTSC:
9133 case VMX_EXIT_RDTSCP:
9134 case VMX_EXIT_MOV_CRX:
9135 case VMX_EXIT_MWAIT:
9136 case VMX_EXIT_MONITOR:
9137 case VMX_EXIT_RDMSR:
9138 case VMX_EXIT_WRMSR:
9139 case VMX_EXIT_MOV_DRX:
9140 case VMX_EXIT_HLT:
9141 case VMX_EXIT_INVD:
9142 case VMX_EXIT_INVLPG:
9143 case VMX_EXIT_RSM:
9144 case VMX_EXIT_PAUSE:
9145 case VMX_EXIT_XDTR_ACCESS:
9146 case VMX_EXIT_TR_ACCESS:
9147 case VMX_EXIT_WBINVD:
9148 case VMX_EXIT_XSETBV:
9149 case VMX_EXIT_RDRAND:
9150 case VMX_EXIT_INVPCID:
9151 case VMX_EXIT_GETSEC:
9152 case VMX_EXIT_RDPMC:
9153 case VMX_EXIT_VMCALL:
9154 case VMX_EXIT_VMCLEAR:
9155 case VMX_EXIT_VMLAUNCH:
9156 case VMX_EXIT_VMPTRLD:
9157 case VMX_EXIT_VMPTRST:
9158 case VMX_EXIT_VMREAD:
9159 case VMX_EXIT_VMRESUME:
9160 case VMX_EXIT_VMWRITE:
9161 case VMX_EXIT_VMXOFF:
9162 case VMX_EXIT_VMXON:
9163 case VMX_EXIT_INVEPT:
9164 case VMX_EXIT_INVVPID:
9165 case VMX_EXIT_VMFUNC:
9166 {
9167 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9168 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9169 AssertRCReturn(rc2, rc2);
9170 if ( pMixedCtx->rip != uRipStart
9171 || pMixedCtx->cs.Sel != uCsStart)
9172 return VINF_EM_DBG_STEPPED;
9173 break;
9174 }
9175 }
9176
9177 /*
9178 * Normal processing.
9179 */
9180#ifdef HMVMX_USE_FUNCTION_TABLE
9181 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9182#else
9183 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9184#endif
9185}
9186
9187
9188#ifdef DEBUG
9189/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9190# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9191 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9192
9193# define HMVMX_ASSERT_PREEMPT_CPUID() \
9194 do \
9195 { \
9196 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9197 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9198 } while (0)
9199
9200# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9201 do { \
9202 AssertPtr(pVCpu); \
9203 AssertPtr(pMixedCtx); \
9204 AssertPtr(pVmxTransient); \
9205 Assert(pVmxTransient->fVMEntryFailed == false); \
9206 Assert(ASMIntAreEnabled()); \
9207 HMVMX_ASSERT_PREEMPT_SAFE(); \
9208 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9209 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9210 HMVMX_ASSERT_PREEMPT_SAFE(); \
9211 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9212 HMVMX_ASSERT_PREEMPT_CPUID(); \
9213 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9214 } while (0)
9215
9216# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9217 do { \
9218 Log4Func(("\n")); \
9219 } while (0)
9220#else /* Release builds */
9221# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9222 do { \
9223 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9224 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9225 } while (0)
9226# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9227#endif
9228
9229
9230/**
9231 * Advances the guest RIP after reading it from the VMCS.
9232 *
9233 * @returns VBox status code.
9234 * @param pVCpu Pointer to the VMCPU.
9235 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9236 * out-of-sync. Make sure to update the required fields
9237 * before using them.
9238 * @param pVmxTransient Pointer to the VMX transient structure.
9239 *
9240 * @remarks No-long-jump zone!!!
9241 */
9242DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9243{
9244 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9245 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9246 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9247 AssertRCReturn(rc, rc);
9248
9249 pMixedCtx->rip += pVmxTransient->cbInstr;
9250 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9251
9252 /*
9253 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9254 * pending debug exception field as it takes care of priority of events.
9255 *
9256 * See Intel spec. 32.2.1 "Debug Exceptions".
9257 */
9258 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9259
9260 return rc;
9261}
9262
9263
9264/**
9265 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9266 * and update error record fields accordingly.
9267 *
9268 * @return VMX_IGS_* return codes.
9269 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9270 * wrong with the guest state.
9271 *
9272 * @param pVM Pointer to the VM.
9273 * @param pVCpu Pointer to the VMCPU.
9274 * @param pCtx Pointer to the guest-CPU state.
9275 *
9276 * @remarks This function assumes our cache of the VMCS controls
9277 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9278 */
9279static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9280{
9281#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9282#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9283 uError = (err); \
9284 break; \
9285 } else do { } while (0)
9286
9287 int rc;
9288 uint32_t uError = VMX_IGS_ERROR;
9289 uint32_t u32Val;
9290 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9291
9292 do
9293 {
9294 /*
9295 * CR0.
9296 */
9297 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9298 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9299 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9300 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9301 if (fUnrestrictedGuest)
9302 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9303
9304 uint32_t u32GuestCR0;
9305 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9306 AssertRCBreak(rc);
9307 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9308 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9309 if ( !fUnrestrictedGuest
9310 && (u32GuestCR0 & X86_CR0_PG)
9311 && !(u32GuestCR0 & X86_CR0_PE))
9312 {
9313 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9314 }
9315
9316 /*
9317 * CR4.
9318 */
9319 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9320 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9321
9322 uint32_t u32GuestCR4;
9323 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9324 AssertRCBreak(rc);
9325 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9326 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9327
9328 /*
9329 * IA32_DEBUGCTL MSR.
9330 */
9331 uint64_t u64Val;
9332 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9333 AssertRCBreak(rc);
9334 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9335 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9336 {
9337 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9338 }
9339 uint64_t u64DebugCtlMsr = u64Val;
9340
9341#ifdef VBOX_STRICT
9342 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9343 AssertRCBreak(rc);
9344 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9345#endif
9346 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9347
9348 /*
9349 * RIP and RFLAGS.
9350 */
9351 uint32_t u32Eflags;
9352#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9353 if (HMVMX_IS_64BIT_HOST_MODE())
9354 {
9355 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9356 AssertRCBreak(rc);
9357 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9358 if ( !fLongModeGuest
9359 || !pCtx->cs.Attr.n.u1Long)
9360 {
9361 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9362 }
9363 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9364 * must be identical if the "IA-32e mode guest" VM-entry
9365 * control is 1 and CS.L is 1. No check applies if the
9366 * CPU supports 64 linear-address bits. */
9367
9368 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9369 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9370 AssertRCBreak(rc);
9371 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9372 VMX_IGS_RFLAGS_RESERVED);
9373 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9374 u32Eflags = u64Val;
9375 }
9376 else
9377#endif
9378 {
9379 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9380 AssertRCBreak(rc);
9381 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9382 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9383 }
9384
9385 if ( fLongModeGuest
9386 || ( fUnrestrictedGuest
9387 && !(u32GuestCR0 & X86_CR0_PE)))
9388 {
9389 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9390 }
9391
9392 uint32_t u32EntryInfo;
9393 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9394 AssertRCBreak(rc);
9395 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9396 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9397 {
9398 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9399 }
9400
9401 /*
9402 * 64-bit checks.
9403 */
9404#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9405 if (HMVMX_IS_64BIT_HOST_MODE())
9406 {
9407 if ( fLongModeGuest
9408 && !fUnrestrictedGuest)
9409 {
9410 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9411 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9412 }
9413
9414 if ( !fLongModeGuest
9415 && (u32GuestCR4 & X86_CR4_PCIDE))
9416 {
9417 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9418 }
9419
9420 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9421 * 51:32 beyond the processor's physical-address width are 0. */
9422
9423 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9424 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9425 {
9426 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9427 }
9428
9429 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9430 AssertRCBreak(rc);
9431 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9432
9433 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9434 AssertRCBreak(rc);
9435 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9436 }
9437#endif
9438
9439 /*
9440 * PERF_GLOBAL MSR.
9441 */
9442 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9443 {
9444 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9445 AssertRCBreak(rc);
9446 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9447 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9448 }
9449
9450 /*
9451 * PAT MSR.
9452 */
9453 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9454 {
9455 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9456 AssertRCBreak(rc);
9457 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9458 for (unsigned i = 0; i < 8; i++)
9459 {
9460 uint8_t u8Val = (u64Val & 0xff);
9461 if ( u8Val != 0 /* UC */
9462 && u8Val != 1 /* WC */
9463 && u8Val != 4 /* WT */
9464 && u8Val != 5 /* WP */
9465 && u8Val != 6 /* WB */
9466 && u8Val != 7 /* UC- */)
9467 {
9468 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9469 }
9470 u64Val >>= 8;
9471 }
9472 }
9473
9474 /*
9475 * EFER MSR.
9476 */
9477 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9478 {
9479 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9480 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9481 AssertRCBreak(rc);
9482 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9483 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9484 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9485 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9486 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9487 || !(u32GuestCR0 & X86_CR0_PG)
9488 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9489 VMX_IGS_EFER_LMA_LME_MISMATCH);
9490 }
9491
9492 /*
9493 * Segment registers.
9494 */
9495 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9496 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9497 if (!(u32Eflags & X86_EFL_VM))
9498 {
9499 /* CS */
9500 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9501 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9502 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9503 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9504 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9505 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9506 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9507 /* CS cannot be loaded with NULL in protected mode. */
9508 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9509 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9510 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9511 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9512 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9513 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9514 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9515 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9516 else
9517 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9518
9519 /* SS */
9520 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9521 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9522 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9523 if ( !(pCtx->cr0 & X86_CR0_PE)
9524 || pCtx->cs.Attr.n.u4Type == 3)
9525 {
9526 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9527 }
9528 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9529 {
9530 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9531 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9532 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9533 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9534 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9535 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9536 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9537 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9538 }
9539
9540 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9541 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9542 {
9543 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9544 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9545 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9546 || pCtx->ds.Attr.n.u4Type > 11
9547 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9548 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9549 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9550 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9551 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9552 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9553 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9554 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9555 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9556 }
9557 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9558 {
9559 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9560 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9561 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9562 || pCtx->es.Attr.n.u4Type > 11
9563 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9564 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9565 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9566 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9567 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9568 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9569 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9570 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9571 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9572 }
9573 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9574 {
9575 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9576 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9577 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9578 || pCtx->fs.Attr.n.u4Type > 11
9579 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9580 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9581 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9582 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9583 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9584 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9585 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9586 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9587 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9588 }
9589 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9590 {
9591 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9592 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9593 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9594 || pCtx->gs.Attr.n.u4Type > 11
9595 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9596 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9597 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9598 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9599 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9600 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9601 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9602 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9603 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9604 }
9605 /* 64-bit capable CPUs. */
9606#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9607 if (HMVMX_IS_64BIT_HOST_MODE())
9608 {
9609 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9610 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9611 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9612 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9613 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9614 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9615 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9616 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9617 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9618 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9619 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9620 }
9621#endif
9622 }
9623 else
9624 {
9625 /* V86 mode checks. */
9626 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9627 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9628 {
9629 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9630 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9631 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9632 }
9633 else
9634 {
9635 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9636 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9637 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9638 }
9639
9640 /* CS */
9641 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9642 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9643 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9644 /* SS */
9645 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9646 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9647 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9648 /* DS */
9649 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9650 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9651 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9652 /* ES */
9653 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9654 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9655 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9656 /* FS */
9657 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9658 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9659 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9660 /* GS */
9661 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9662 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9663 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9664 /* 64-bit capable CPUs. */
9665#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9666 if (HMVMX_IS_64BIT_HOST_MODE())
9667 {
9668 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9669 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9670 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9671 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9672 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9673 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9674 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9675 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9676 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9677 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9678 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9679 }
9680#endif
9681 }
9682
9683 /*
9684 * TR.
9685 */
9686 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9687 /* 64-bit capable CPUs. */
9688#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9689 if (HMVMX_IS_64BIT_HOST_MODE())
9690 {
9691 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9692 }
9693#endif
9694 if (fLongModeGuest)
9695 {
9696 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9697 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9698 }
9699 else
9700 {
9701 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9702 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9703 VMX_IGS_TR_ATTR_TYPE_INVALID);
9704 }
9705 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9706 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9707 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9708 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9709 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9710 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9711 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9712 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9713
9714 /*
9715 * GDTR and IDTR.
9716 */
9717#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9718 if (HMVMX_IS_64BIT_HOST_MODE())
9719 {
9720 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9721 AssertRCBreak(rc);
9722 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9723
9724 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9725 AssertRCBreak(rc);
9726 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9727 }
9728#endif
9729
9730 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9731 AssertRCBreak(rc);
9732 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9733
9734 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9735 AssertRCBreak(rc);
9736 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9737
9738 /*
9739 * Guest Non-Register State.
9740 */
9741 /* Activity State. */
9742 uint32_t u32ActivityState;
9743 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9744 AssertRCBreak(rc);
9745 HMVMX_CHECK_BREAK( !u32ActivityState
9746 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9747 VMX_IGS_ACTIVITY_STATE_INVALID);
9748 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9749 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9750 uint32_t u32IntrState;
9751 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9752 AssertRCBreak(rc);
9753 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9754 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9755 {
9756 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9757 }
9758
9759 /** @todo Activity state and injecting interrupts. Left as a todo since we
9760 * currently don't use activity states but ACTIVE. */
9761
9762 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9763 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9764
9765 /* Guest interruptibility-state. */
9766 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9767 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9768 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9769 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9770 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9771 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9772 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9773 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9774 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9775 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9776 {
9777 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9778 {
9779 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9780 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9781 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9782 }
9783 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9784 {
9785 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9786 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9787 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9788 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9789 }
9790 }
9791 /** @todo Assumes the processor is not in SMM. */
9792 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9793 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9794 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9795 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9796 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9797 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9798 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9799 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9800 {
9801 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9802 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9803 }
9804
9805 /* Pending debug exceptions. */
9806 if (HMVMX_IS_64BIT_HOST_MODE())
9807 {
9808 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9809 AssertRCBreak(rc);
9810 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9811 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9812 u32Val = u64Val; /* For pending debug exceptions checks below. */
9813 }
9814 else
9815 {
9816 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9817 AssertRCBreak(rc);
9818 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9819 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9820 }
9821
9822 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9823 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9824 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9825 {
9826 if ( (u32Eflags & X86_EFL_TF)
9827 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9828 {
9829 /* Bit 14 is PendingDebug.BS. */
9830 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9831 }
9832 if ( !(u32Eflags & X86_EFL_TF)
9833 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9834 {
9835 /* Bit 14 is PendingDebug.BS. */
9836 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9837 }
9838 }
9839
9840 /* VMCS link pointer. */
9841 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9842 AssertRCBreak(rc);
9843 if (u64Val != UINT64_C(0xffffffffffffffff))
9844 {
9845 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9846 /** @todo Bits beyond the processor's physical-address width MBZ. */
9847 /** @todo 32-bit located in memory referenced by value of this field (as a
9848 * physical address) must contain the processor's VMCS revision ID. */
9849 /** @todo SMM checks. */
9850 }
9851
9852 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9853 * not using Nested Paging? */
9854 if ( pVM->hm.s.fNestedPaging
9855 && !fLongModeGuest
9856 && CPUMIsGuestInPAEModeEx(pCtx))
9857 {
9858 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9859 AssertRCBreak(rc);
9860 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9861
9862 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9863 AssertRCBreak(rc);
9864 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9865
9866 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9867 AssertRCBreak(rc);
9868 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9869
9870 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9871 AssertRCBreak(rc);
9872 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9873 }
9874
9875 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9876 if (uError == VMX_IGS_ERROR)
9877 uError = VMX_IGS_REASON_NOT_FOUND;
9878 } while (0);
9879
9880 pVCpu->hm.s.u32HMError = uError;
9881 return uError;
9882
9883#undef HMVMX_ERROR_BREAK
9884#undef HMVMX_CHECK_BREAK
9885}
9886
9887/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9888/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9889/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9890
9891/** @name VM-exit handlers.
9892 * @{
9893 */
9894
9895/**
9896 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9897 */
9898HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9899{
9900 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9901 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9902 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9903 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9904 return VINF_SUCCESS;
9905 return VINF_EM_RAW_INTERRUPT;
9906}
9907
9908
9909/**
9910 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9911 */
9912HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9913{
9914 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9915 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9916
9917 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9918 AssertRCReturn(rc, rc);
9919
9920 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9921 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9922 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9923 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9924
9925 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9926 {
9927 /*
9928 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9929 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9930 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9931 *
9932 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9933 */
9934 VMXDispatchHostNmi();
9935 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9936 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9937 return VINF_SUCCESS;
9938 }
9939
9940 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9941 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9942 if (RT_UNLIKELY(rc != VINF_SUCCESS))
9943 {
9944 if (rc == VINF_HM_DOUBLE_FAULT)
9945 rc = VINF_SUCCESS;
9946 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9947 return rc;
9948 }
9949
9950 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9951 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9952 switch (uIntType)
9953 {
9954 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9955 Assert(uVector == X86_XCPT_DB);
9956 /* no break */
9957 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9958 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9959 /* no break */
9960 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9961 {
9962 switch (uVector)
9963 {
9964 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9965 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9966 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9967 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9968 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9969 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9970#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9971 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9972 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9973 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9974 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9975 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9976 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9977 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9978 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9979 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9980 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9981 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9982 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9983#endif
9984 default:
9985 {
9986 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9987 AssertRCReturn(rc, rc);
9988
9989 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9990 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9991 {
9992 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9993 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9994 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9995
9996 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9997 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9998 AssertRCReturn(rc, rc);
9999 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10000 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10001 0 /* GCPtrFaultAddress */);
10002 AssertRCReturn(rc, rc);
10003 }
10004 else
10005 {
10006 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10007 pVCpu->hm.s.u32HMError = uVector;
10008 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10009 }
10010 break;
10011 }
10012 }
10013 break;
10014 }
10015
10016 default:
10017 {
10018 pVCpu->hm.s.u32HMError = uExitIntInfo;
10019 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10020 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10021 break;
10022 }
10023 }
10024 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10025 return rc;
10026}
10027
10028
10029/**
10030 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10031 */
10032HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10033{
10034 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10035
10036 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
10037 hmR0VmxClearIntWindowExitVmcs(pVCpu);
10038
10039 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10040 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
10041 return VINF_SUCCESS;
10042}
10043
10044
10045/**
10046 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
10047 */
10048HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10049{
10050 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10051 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
10052 {
10053 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
10054 HMVMX_RETURN_UNEXPECTED_EXIT();
10055 }
10056
10057 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
10058
10059 /*
10060 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
10061 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
10062 */
10063 uint32_t uIntrState = 0;
10064 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10065 AssertRCReturn(rc, rc);
10066
10067 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
10068 if ( fBlockSti
10069 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
10070 {
10071 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10072 }
10073
10074 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
10075 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
10076
10077 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10078 return VINF_SUCCESS;
10079}
10080
10081
10082/**
10083 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10084 */
10085HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10086{
10087 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10088 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10089 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10090}
10091
10092
10093/**
10094 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10095 */
10096HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10097{
10098 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10099 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10100 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10101}
10102
10103
10104/**
10105 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10106 */
10107HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10108{
10109 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10110 PVM pVM = pVCpu->CTX_SUFF(pVM);
10111 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10112 if (RT_LIKELY(rc == VINF_SUCCESS))
10113 {
10114 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10115 Assert(pVmxTransient->cbInstr == 2);
10116 }
10117 else
10118 {
10119 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10120 rc = VERR_EM_INTERPRETER;
10121 }
10122 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10123 return rc;
10124}
10125
10126
10127/**
10128 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10129 */
10130HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10131{
10132 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10133 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10134 AssertRCReturn(rc, rc);
10135
10136 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10137 return VINF_EM_RAW_EMULATE_INSTR;
10138
10139 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10140 HMVMX_RETURN_UNEXPECTED_EXIT();
10141}
10142
10143
10144/**
10145 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10146 */
10147HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10148{
10149 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10150 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10151 AssertRCReturn(rc, rc);
10152
10153 PVM pVM = pVCpu->CTX_SUFF(pVM);
10154 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10155 if (RT_LIKELY(rc == VINF_SUCCESS))
10156 {
10157 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10158 Assert(pVmxTransient->cbInstr == 2);
10159 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10160 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10161 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10162 }
10163 else
10164 rc = VERR_EM_INTERPRETER;
10165 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10166 return rc;
10167}
10168
10169
10170/**
10171 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10172 */
10173HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10174{
10175 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10176 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10177 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10178 AssertRCReturn(rc, rc);
10179
10180 PVM pVM = pVCpu->CTX_SUFF(pVM);
10181 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10182 if (RT_LIKELY(rc == VINF_SUCCESS))
10183 {
10184 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10185 Assert(pVmxTransient->cbInstr == 3);
10186 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10187 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10188 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10189 }
10190 else
10191 {
10192 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10193 rc = VERR_EM_INTERPRETER;
10194 }
10195 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10196 return rc;
10197}
10198
10199
10200/**
10201 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10202 */
10203HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10204{
10205 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10206 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10207 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10208 AssertRCReturn(rc, rc);
10209
10210 PVM pVM = pVCpu->CTX_SUFF(pVM);
10211 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10212 if (RT_LIKELY(rc == VINF_SUCCESS))
10213 {
10214 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10215 Assert(pVmxTransient->cbInstr == 2);
10216 }
10217 else
10218 {
10219 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10220 rc = VERR_EM_INTERPRETER;
10221 }
10222 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10223 return rc;
10224}
10225
10226
10227/**
10228 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10229 */
10230HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10231{
10232 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10233
10234 int rc = VERR_NOT_SUPPORTED;
10235 if (GIMAreHypercallsEnabled(pVCpu))
10236 {
10237 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10238 AssertRCReturn(rc, rc);
10239
10240 rc = GIMHypercall(pVCpu, pMixedCtx);
10241 }
10242 if (rc != VINF_SUCCESS)
10243 {
10244 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10245 rc = VINF_SUCCESS;
10246 }
10247
10248 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10249 return rc;
10250}
10251
10252
10253/**
10254 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10255 */
10256HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10257{
10258 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10259 PVM pVM = pVCpu->CTX_SUFF(pVM);
10260 Assert(!pVM->hm.s.fNestedPaging);
10261
10262 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10263 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10264 AssertRCReturn(rc, rc);
10265
10266 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10267 rc = VBOXSTRICTRC_VAL(rc2);
10268 if (RT_LIKELY(rc == VINF_SUCCESS))
10269 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10270 else
10271 {
10272 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10273 pVmxTransient->uExitQualification, rc));
10274 }
10275 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10276 return rc;
10277}
10278
10279
10280/**
10281 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10282 */
10283HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10284{
10285 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10286 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10287 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10288 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10289 AssertRCReturn(rc, rc);
10290
10291 PVM pVM = pVCpu->CTX_SUFF(pVM);
10292 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10293 if (RT_LIKELY(rc == VINF_SUCCESS))
10294 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10295 else
10296 {
10297 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10298 rc = VERR_EM_INTERPRETER;
10299 }
10300 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10301 return rc;
10302}
10303
10304
10305/**
10306 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10307 */
10308HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10309{
10310 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10311 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10312 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10313 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10314 AssertRCReturn(rc, rc);
10315
10316 PVM pVM = pVCpu->CTX_SUFF(pVM);
10317 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10318 rc = VBOXSTRICTRC_VAL(rc2);
10319 if (RT_LIKELY( rc == VINF_SUCCESS
10320 || rc == VINF_EM_HALT))
10321 {
10322 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10323 AssertRCReturn(rc3, rc3);
10324
10325 if ( rc == VINF_EM_HALT
10326 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10327 {
10328 rc = VINF_SUCCESS;
10329 }
10330 }
10331 else
10332 {
10333 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10334 rc = VERR_EM_INTERPRETER;
10335 }
10336 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10337 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10338 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10339 return rc;
10340}
10341
10342
10343/**
10344 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10345 */
10346HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10347{
10348 /*
10349 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10350 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10351 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10352 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10353 */
10354 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10355 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10356 HMVMX_RETURN_UNEXPECTED_EXIT();
10357}
10358
10359
10360/**
10361 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10362 */
10363HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10364{
10365 /*
10366 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10367 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10368 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10369 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10370 */
10371 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10372 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10373 HMVMX_RETURN_UNEXPECTED_EXIT();
10374}
10375
10376
10377/**
10378 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10379 */
10380HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10381{
10382 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10383 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10384 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10385 HMVMX_RETURN_UNEXPECTED_EXIT();
10386}
10387
10388
10389/**
10390 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10391 */
10392HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10393{
10394 /*
10395 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10396 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10397 * See Intel spec. 25.3 "Other Causes of VM-exits".
10398 */
10399 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10400 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10401 HMVMX_RETURN_UNEXPECTED_EXIT();
10402}
10403
10404
10405/**
10406 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10407 * VM-exit.
10408 */
10409HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10410{
10411 /*
10412 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10413 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10414 *
10415 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10416 * See Intel spec. "23.8 Restrictions on VMX operation".
10417 */
10418 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10419 return VINF_SUCCESS;
10420}
10421
10422
10423/**
10424 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10425 * VM-exit.
10426 */
10427HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10428{
10429 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10430 return VINF_EM_RESET;
10431}
10432
10433
10434/**
10435 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10436 */
10437HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10438{
10439 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10440 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10441 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10442 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10443 AssertRCReturn(rc, rc);
10444
10445 pMixedCtx->rip++;
10446 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10447 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10448 rc = VINF_SUCCESS;
10449 else
10450 rc = VINF_EM_HALT;
10451
10452 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10453 if (rc != VINF_SUCCESS)
10454 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHltToR3);
10455 return rc;
10456}
10457
10458
10459/**
10460 * VM-exit handler for instructions that result in a #UD exception delivered to
10461 * the guest.
10462 */
10463HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10464{
10465 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10466 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10467 return VINF_SUCCESS;
10468}
10469
10470
10471/**
10472 * VM-exit handler for expiry of the VMX preemption timer.
10473 */
10474HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10475{
10476 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10477
10478 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10479 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10480
10481 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10482 PVM pVM = pVCpu->CTX_SUFF(pVM);
10483 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10484 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10485 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10486}
10487
10488
10489/**
10490 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10491 */
10492HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10493{
10494 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10495
10496 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10497 /** @todo check if XSETBV is supported by the recompiler. */
10498 return VERR_EM_INTERPRETER;
10499}
10500
10501
10502/**
10503 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10504 */
10505HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10506{
10507 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10508
10509 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10510 /** @todo implement EMInterpretInvpcid() */
10511 return VERR_EM_INTERPRETER;
10512}
10513
10514
10515/**
10516 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10517 * Error VM-exit.
10518 */
10519HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10520{
10521 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10522 AssertRCReturn(rc, rc);
10523
10524 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10525 AssertRCReturn(rc, rc);
10526
10527 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10528 NOREF(uInvalidReason);
10529
10530#ifdef VBOX_STRICT
10531 uint32_t uIntrState;
10532 HMVMXHCUINTREG uHCReg;
10533 uint64_t u64Val;
10534 uint32_t u32Val;
10535
10536 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10537 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10538 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10539 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10540 AssertRCReturn(rc, rc);
10541
10542 Log4(("uInvalidReason %u\n", uInvalidReason));
10543 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10544 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10545 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10546 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10547
10548 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10549 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10550 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10551 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10552 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10553 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10554 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10555 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10556 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10557 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10558 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10559 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10560#else
10561 NOREF(pVmxTransient);
10562#endif
10563
10564 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10565 return VERR_VMX_INVALID_GUEST_STATE;
10566}
10567
10568
10569/**
10570 * VM-exit handler for VM-entry failure due to an MSR-load
10571 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10572 */
10573HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10574{
10575 NOREF(pVmxTransient);
10576 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10577 HMVMX_RETURN_UNEXPECTED_EXIT();
10578}
10579
10580
10581/**
10582 * VM-exit handler for VM-entry failure due to a machine-check event
10583 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10584 */
10585HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10586{
10587 NOREF(pVmxTransient);
10588 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10589 HMVMX_RETURN_UNEXPECTED_EXIT();
10590}
10591
10592
10593/**
10594 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10595 * theory.
10596 */
10597HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10598{
10599 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10600 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10601 return VERR_VMX_UNDEFINED_EXIT_CODE;
10602}
10603
10604
10605/**
10606 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10607 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10608 * Conditional VM-exit.
10609 */
10610HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10611{
10612 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10613
10614 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10615 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10616 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10617 return VERR_EM_INTERPRETER;
10618 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10619 HMVMX_RETURN_UNEXPECTED_EXIT();
10620}
10621
10622
10623/**
10624 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10625 */
10626HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10627{
10628 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10629
10630 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10631 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10632 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10633 return VERR_EM_INTERPRETER;
10634 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10635 HMVMX_RETURN_UNEXPECTED_EXIT();
10636}
10637
10638
10639/**
10640 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10641 */
10642HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10643{
10644 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10645
10646 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10647 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10648 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10649 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10650 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10651 {
10652 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10653 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10654 }
10655 AssertRCReturn(rc, rc);
10656 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10657
10658#ifdef VBOX_STRICT
10659 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10660 {
10661 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10662 && pMixedCtx->ecx != MSR_K6_EFER)
10663 {
10664 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10665 HMVMX_RETURN_UNEXPECTED_EXIT();
10666 }
10667# if HC_ARCH_BITS == 64
10668 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10669 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10670 {
10671 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10672 HMVMX_RETURN_UNEXPECTED_EXIT();
10673 }
10674# endif
10675 }
10676#endif
10677
10678 PVM pVM = pVCpu->CTX_SUFF(pVM);
10679 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10680 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10681 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10682 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10683 if (RT_LIKELY(rc == VINF_SUCCESS))
10684 {
10685 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10686 Assert(pVmxTransient->cbInstr == 2);
10687 }
10688 return rc;
10689}
10690
10691
10692/**
10693 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10694 */
10695HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10696{
10697 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10698 PVM pVM = pVCpu->CTX_SUFF(pVM);
10699 int rc = VINF_SUCCESS;
10700
10701 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10702 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10703 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10704 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10705 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10706 {
10707 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10708 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10709 }
10710 AssertRCReturn(rc, rc);
10711 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10712
10713 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10714 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10715 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10716
10717 if (RT_LIKELY(rc == VINF_SUCCESS))
10718 {
10719 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10720
10721 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10722 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10723 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10724 {
10725 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10726 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10727 EMInterpretWrmsr() changes it. */
10728 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10729 }
10730 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10731 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10732 else if (pMixedCtx->ecx == MSR_K6_EFER)
10733 {
10734 /*
10735 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10736 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10737 * the other bits as well, SCE and NXE. See @bugref{7368}.
10738 */
10739 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10740 }
10741
10742 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10743 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10744 {
10745 switch (pMixedCtx->ecx)
10746 {
10747 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10748 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10749 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10750 case MSR_K8_FS_BASE: /* no break */
10751 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10752 case MSR_K6_EFER: /* already handled above */ break;
10753 default:
10754 {
10755 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10756 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10757#if HC_ARCH_BITS == 64
10758 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10759 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10760#endif
10761 break;
10762 }
10763 }
10764 }
10765#ifdef VBOX_STRICT
10766 else
10767 {
10768 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10769 switch (pMixedCtx->ecx)
10770 {
10771 case MSR_IA32_SYSENTER_CS:
10772 case MSR_IA32_SYSENTER_EIP:
10773 case MSR_IA32_SYSENTER_ESP:
10774 case MSR_K8_FS_BASE:
10775 case MSR_K8_GS_BASE:
10776 {
10777 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10778 HMVMX_RETURN_UNEXPECTED_EXIT();
10779 }
10780
10781 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10782 default:
10783 {
10784 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10785 {
10786 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10787 if (pMixedCtx->ecx != MSR_K6_EFER)
10788 {
10789 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10790 pMixedCtx->ecx));
10791 HMVMX_RETURN_UNEXPECTED_EXIT();
10792 }
10793 }
10794
10795#if HC_ARCH_BITS == 64
10796 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10797 {
10798 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10799 HMVMX_RETURN_UNEXPECTED_EXIT();
10800 }
10801#endif
10802 break;
10803 }
10804 }
10805 }
10806#endif /* VBOX_STRICT */
10807 }
10808 return rc;
10809}
10810
10811
10812/**
10813 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10814 */
10815HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10816{
10817 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10818
10819 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10820 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10821 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10822 return VERR_EM_INTERPRETER;
10823 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10824 HMVMX_RETURN_UNEXPECTED_EXIT();
10825}
10826
10827
10828/**
10829 * VM-exit handler for when the TPR value is lowered below the specified
10830 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10831 */
10832HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10833{
10834 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10835 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10836
10837 /*
10838 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10839 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10840 * resume guest execution.
10841 */
10842 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10843 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10844 return VINF_SUCCESS;
10845}
10846
10847
10848/**
10849 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10850 * VM-exit.
10851 *
10852 * @retval VINF_SUCCESS when guest execution can continue.
10853 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10854 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10855 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10856 * recompiler.
10857 */
10858HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10859{
10860 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10861 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10862 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10863 AssertRCReturn(rc, rc);
10864
10865 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10866 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10867 PVM pVM = pVCpu->CTX_SUFF(pVM);
10868 switch (uAccessType)
10869 {
10870 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10871 {
10872#if 0
10873 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10874 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10875#else
10876 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10877 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10878 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10879#endif
10880 AssertRCReturn(rc, rc);
10881
10882 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10883 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10884 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10885 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10886
10887 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10888 {
10889 case 0: /* CR0 */
10890 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10891 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10892 break;
10893 case 2: /* CR2 */
10894 /* Nothing to do here, CR2 it's not part of the VMCS. */
10895 break;
10896 case 3: /* CR3 */
10897 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10898 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10899 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10900 break;
10901 case 4: /* CR4 */
10902 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10903 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10904 break;
10905 case 8: /* CR8 */
10906 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10907 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10908 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10909 break;
10910 default:
10911 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10912 break;
10913 }
10914
10915 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10916 break;
10917 }
10918
10919 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10920 {
10921 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10922 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10923 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10924 AssertRCReturn(rc, rc);
10925 Assert( !pVM->hm.s.fNestedPaging
10926 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10927 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10928
10929 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10930 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10931 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10932
10933 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10934 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10935 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10936 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10937 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10938 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10939 break;
10940 }
10941
10942 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10943 {
10944 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10945 AssertRCReturn(rc, rc);
10946 rc = EMInterpretCLTS(pVM, pVCpu);
10947 AssertRCReturn(rc, rc);
10948 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10949 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10950 Log4(("CRX CLTS write rc=%d\n", rc));
10951 break;
10952 }
10953
10954 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10955 {
10956 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10957 AssertRCReturn(rc, rc);
10958 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10959 if (RT_LIKELY(rc == VINF_SUCCESS))
10960 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10961 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10962 Log4(("CRX LMSW write rc=%d\n", rc));
10963 break;
10964 }
10965
10966 default:
10967 {
10968 AssertMsgFailed(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType));
10969 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10970 }
10971 }
10972
10973 /* Validate possible error codes. */
10974 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10975 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10976 if (RT_SUCCESS(rc))
10977 {
10978 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10979 AssertRCReturn(rc2, rc2);
10980 }
10981
10982 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10983 return rc;
10984}
10985
10986
10987/**
10988 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10989 * VM-exit.
10990 */
10991HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10992{
10993 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10994 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10995
10996 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10997 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10998 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10999 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
11000 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
11001 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
11002 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
11003 AssertRCReturn(rc2, rc2);
11004
11005 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
11006 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
11007 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
11008 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
11009 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
11010 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
11011 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11012 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
11013
11014 /* I/O operation lookup arrays. */
11015 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
11016 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
11017
11018 VBOXSTRICTRC rcStrict;
11019 uint32_t const cbValue = s_aIOSizes[uIOWidth];
11020 uint32_t const cbInstr = pVmxTransient->cbInstr;
11021 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
11022 PVM pVM = pVCpu->CTX_SUFF(pVM);
11023 if (fIOString)
11024 {
11025#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
11026 /*
11027 * INS/OUTS - I/O String instruction.
11028 *
11029 * Use instruction-information if available, otherwise fall back on
11030 * interpreting the instruction.
11031 */
11032 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11033 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
11034 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
11035 {
11036 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11037 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11038 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11039 AssertRCReturn(rc2, rc2);
11040 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
11041 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
11042 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
11043 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
11044 if (fIOWrite)
11045 {
11046 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
11047 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
11048 }
11049 else
11050 {
11051 /*
11052 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
11053 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
11054 * See Intel Instruction spec. for "INS".
11055 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
11056 */
11057 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
11058 }
11059 }
11060 else
11061 {
11062 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11063 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11064 AssertRCReturn(rc2, rc2);
11065 rcStrict = IEMExecOne(pVCpu);
11066 }
11067 /** @todo IEM needs to be setting these flags somehow. */
11068 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11069 fUpdateRipAlready = true;
11070#else
11071 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11072 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
11073 if (RT_SUCCESS(rcStrict))
11074 {
11075 if (fIOWrite)
11076 {
11077 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11078 (DISCPUMODE)pDis->uAddrMode, cbValue);
11079 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
11080 }
11081 else
11082 {
11083 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11084 (DISCPUMODE)pDis->uAddrMode, cbValue);
11085 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11086 }
11087 }
11088 else
11089 {
11090 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
11091 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11092 }
11093#endif
11094 }
11095 else
11096 {
11097 /*
11098 * IN/OUT - I/O instruction.
11099 */
11100 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11101 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11102 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11103 if (fIOWrite)
11104 {
11105 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11106 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11107 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11108 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11109 }
11110 else
11111 {
11112 uint32_t u32Result = 0;
11113 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11114 if (IOM_SUCCESS(rcStrict))
11115 {
11116 /* Save result of I/O IN instr. in AL/AX/EAX. */
11117 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11118 }
11119 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11120 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11121 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11122 }
11123 }
11124
11125 if (IOM_SUCCESS(rcStrict))
11126 {
11127 if (!fUpdateRipAlready)
11128 {
11129 pMixedCtx->rip += cbInstr;
11130 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11131 }
11132
11133 /*
11134 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11135 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11136 */
11137 if (fIOString)
11138 {
11139 /** @todo Single-step for INS/OUTS with REP prefix? */
11140 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11141 }
11142 else if (fStepping)
11143 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11144
11145 /*
11146 * If any I/O breakpoints are armed, we need to check if one triggered
11147 * and take appropriate action.
11148 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11149 */
11150 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11151 AssertRCReturn(rc2, rc2);
11152
11153 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11154 * execution engines about whether hyper BPs and such are pending. */
11155 uint32_t const uDr7 = pMixedCtx->dr[7];
11156 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11157 && X86_DR7_ANY_RW_IO(uDr7)
11158 && (pMixedCtx->cr4 & X86_CR4_DE))
11159 || DBGFBpIsHwIoArmed(pVM)))
11160 {
11161 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11162
11163 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11164 VMMRZCallRing3Disable(pVCpu);
11165 HM_DISABLE_PREEMPT_IF_NEEDED();
11166
11167 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11168
11169 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11170 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11171 {
11172 /* Raise #DB. */
11173 if (fIsGuestDbgActive)
11174 ASMSetDR6(pMixedCtx->dr[6]);
11175 if (pMixedCtx->dr[7] != uDr7)
11176 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11177
11178 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11179 }
11180 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11181 else if ( rcStrict2 != VINF_SUCCESS
11182 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11183 rcStrict = rcStrict2;
11184
11185 HM_RESTORE_PREEMPT_IF_NEEDED();
11186 VMMRZCallRing3Enable(pVCpu);
11187 }
11188 }
11189
11190#ifdef DEBUG
11191 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11192 Assert(!fIOWrite);
11193 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11194 Assert(fIOWrite);
11195 else
11196 {
11197 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11198 * statuses, that the VMM device and some others may return. See
11199 * IOM_SUCCESS() for guidance. */
11200 AssertMsg( RT_FAILURE(rcStrict)
11201 || rcStrict == VINF_SUCCESS
11202 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11203 || rcStrict == VINF_EM_DBG_BREAKPOINT
11204 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11205 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11206 }
11207#endif
11208
11209 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11210 return VBOXSTRICTRC_TODO(rcStrict);
11211}
11212
11213
11214/**
11215 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11216 * VM-exit.
11217 */
11218HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11219{
11220 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11221
11222 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11223 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11224 AssertRCReturn(rc, rc);
11225 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11226 {
11227 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11228 AssertRCReturn(rc, rc);
11229 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11230 {
11231 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11232
11233 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11234 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11235
11236 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11237 Assert(!pVCpu->hm.s.Event.fPending);
11238 pVCpu->hm.s.Event.fPending = true;
11239 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11240 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11241 AssertRCReturn(rc, rc);
11242 if (fErrorCodeValid)
11243 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11244 else
11245 pVCpu->hm.s.Event.u32ErrCode = 0;
11246 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11247 && uVector == X86_XCPT_PF)
11248 {
11249 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11250 }
11251
11252 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11253 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11254 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11255 }
11256 }
11257
11258 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11259 * emulation. */
11260 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11261 return VERR_EM_INTERPRETER;
11262}
11263
11264
11265/**
11266 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11267 */
11268HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11269{
11270 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11271 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11272 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11273 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11274 AssertRCReturn(rc, rc);
11275 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11276 return VINF_EM_DBG_STEPPED;
11277}
11278
11279
11280/**
11281 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11282 */
11283HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11284{
11285 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11286
11287 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11288 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11289 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11290 {
11291 if (rc == VINF_HM_DOUBLE_FAULT)
11292 rc = VINF_SUCCESS;
11293 return rc;
11294 }
11295
11296#if 0
11297 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11298 * just sync the whole thing. */
11299 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11300#else
11301 /* Aggressive state sync. for now. */
11302 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11303 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11304 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11305#endif
11306 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11307 AssertRCReturn(rc, rc);
11308
11309 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11310 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11311 switch (uAccessType)
11312 {
11313 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11314 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11315 {
11316 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11317 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11318 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11319
11320 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11321 GCPhys &= PAGE_BASE_GC_MASK;
11322 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11323 PVM pVM = pVCpu->CTX_SUFF(pVM);
11324 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11325 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11326
11327 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11328 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11329 CPUMCTX2CORE(pMixedCtx), GCPhys);
11330 rc = VBOXSTRICTRC_VAL(rc2);
11331 Log4(("ApicAccess rc=%d\n", rc));
11332 if ( rc == VINF_SUCCESS
11333 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11334 || rc == VERR_PAGE_NOT_PRESENT)
11335 {
11336 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11337 | HM_CHANGED_GUEST_RSP
11338 | HM_CHANGED_GUEST_RFLAGS
11339 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11340 rc = VINF_SUCCESS;
11341 }
11342 break;
11343 }
11344
11345 default:
11346 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11347 rc = VINF_EM_RAW_EMULATE_INSTR;
11348 break;
11349 }
11350
11351 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11352 if (rc != VINF_SUCCESS)
11353 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccessToR3);
11354 return rc;
11355}
11356
11357
11358/**
11359 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11360 * VM-exit.
11361 */
11362HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11363{
11364 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11365
11366 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11367 if (pVmxTransient->fWasGuestDebugStateActive)
11368 {
11369 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11370 HMVMX_RETURN_UNEXPECTED_EXIT();
11371 }
11372
11373 int rc = VERR_INTERNAL_ERROR_5;
11374 if ( !DBGFIsStepping(pVCpu)
11375 && !pVCpu->hm.s.fSingleInstruction
11376 && !pVmxTransient->fWasHyperDebugStateActive)
11377 {
11378 /* Don't intercept MOV DRx and #DB any more. */
11379 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11380 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11381 AssertRCReturn(rc, rc);
11382
11383 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11384 {
11385#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11386 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11387 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11388 AssertRCReturn(rc, rc);
11389#endif
11390 }
11391
11392 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11393 VMMRZCallRing3Disable(pVCpu);
11394 HM_DISABLE_PREEMPT_IF_NEEDED();
11395
11396 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11397 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11398 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11399
11400 HM_RESTORE_PREEMPT_IF_NEEDED();
11401 VMMRZCallRing3Enable(pVCpu);
11402
11403#ifdef VBOX_WITH_STATISTICS
11404 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11405 AssertRCReturn(rc, rc);
11406 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11407 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11408 else
11409 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11410#endif
11411 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11412 return VINF_SUCCESS;
11413 }
11414
11415 /*
11416 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11417 * Update the segment registers and DR7 from the CPU.
11418 */
11419 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11420 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11421 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11422 AssertRCReturn(rc, rc);
11423 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11424
11425 PVM pVM = pVCpu->CTX_SUFF(pVM);
11426 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11427 {
11428 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11429 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11430 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11431 if (RT_SUCCESS(rc))
11432 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11433 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11434 }
11435 else
11436 {
11437 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11438 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11439 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11440 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11441 }
11442
11443 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11444 if (RT_SUCCESS(rc))
11445 {
11446 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11447 AssertRCReturn(rc2, rc2);
11448 }
11449 return rc;
11450}
11451
11452
11453/**
11454 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11455 * Conditional VM-exit.
11456 */
11457HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11458{
11459 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11460 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11461
11462 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11463 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11464 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11465 {
11466 if (rc == VINF_HM_DOUBLE_FAULT)
11467 rc = VINF_SUCCESS;
11468 return rc;
11469 }
11470
11471 RTGCPHYS GCPhys = 0;
11472 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11473
11474#if 0
11475 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11476#else
11477 /* Aggressive state sync. for now. */
11478 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11479 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11480 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11481#endif
11482 AssertRCReturn(rc, rc);
11483
11484 /*
11485 * If we succeed, resume guest execution.
11486 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11487 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11488 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11489 * weird case. See @bugref{6043}.
11490 */
11491 PVM pVM = pVCpu->CTX_SUFF(pVM);
11492 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11493 rc = VBOXSTRICTRC_VAL(rc2);
11494 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11495 if ( rc == VINF_SUCCESS
11496 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11497 || rc == VERR_PAGE_NOT_PRESENT)
11498 {
11499 /* Successfully handled MMIO operation. */
11500 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11501 | HM_CHANGED_GUEST_RSP
11502 | HM_CHANGED_GUEST_RFLAGS
11503 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11504 rc = VINF_SUCCESS;
11505 }
11506 return rc;
11507}
11508
11509
11510/**
11511 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11512 * VM-exit.
11513 */
11514HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11515{
11516 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11517 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11518
11519 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11520 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11521 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11522 {
11523 if (rc == VINF_HM_DOUBLE_FAULT)
11524 rc = VINF_SUCCESS;
11525 return rc;
11526 }
11527
11528 RTGCPHYS GCPhys = 0;
11529 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11530 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11531#if 0
11532 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11533#else
11534 /* Aggressive state sync. for now. */
11535 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11536 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11537 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11538#endif
11539 AssertRCReturn(rc, rc);
11540
11541 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11542 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11543
11544 RTGCUINT uErrorCode = 0;
11545 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11546 uErrorCode |= X86_TRAP_PF_ID;
11547 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11548 uErrorCode |= X86_TRAP_PF_RW;
11549 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11550 uErrorCode |= X86_TRAP_PF_P;
11551
11552 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11553
11554 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11555 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11556
11557 /* Handle the pagefault trap for the nested shadow table. */
11558 PVM pVM = pVCpu->CTX_SUFF(pVM);
11559 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11560 TRPMResetTrap(pVCpu);
11561
11562 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11563 if ( rc == VINF_SUCCESS
11564 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11565 || rc == VERR_PAGE_NOT_PRESENT)
11566 {
11567 /* Successfully synced our nested page tables. */
11568 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11569 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11570 | HM_CHANGED_GUEST_RSP
11571 | HM_CHANGED_GUEST_RFLAGS);
11572 return VINF_SUCCESS;
11573 }
11574
11575 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11576 return rc;
11577}
11578
11579/** @} */
11580
11581/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11582/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11583/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11584
11585/** @name VM-exit exception handlers.
11586 * @{
11587 */
11588
11589/**
11590 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11591 */
11592static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11593{
11594 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11595 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11596
11597 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11598 AssertRCReturn(rc, rc);
11599
11600 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11601 {
11602 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11603 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11604
11605 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11606 * provides VM-exit instruction length. If this causes problem later,
11607 * disassemble the instruction like it's done on AMD-V. */
11608 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11609 AssertRCReturn(rc2, rc2);
11610 return rc;
11611 }
11612
11613 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11614 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11615 return rc;
11616}
11617
11618
11619/**
11620 * VM-exit exception handler for #BP (Breakpoint exception).
11621 */
11622static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11623{
11624 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11625 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11626
11627 /** @todo Try optimize this by not saving the entire guest state unless
11628 * really needed. */
11629 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11630 AssertRCReturn(rc, rc);
11631
11632 PVM pVM = pVCpu->CTX_SUFF(pVM);
11633 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11634 if (rc == VINF_EM_RAW_GUEST_TRAP)
11635 {
11636 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11637 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11638 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11639 AssertRCReturn(rc, rc);
11640
11641 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11642 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11643 }
11644
11645 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11646 return rc;
11647}
11648
11649
11650/**
11651 * VM-exit exception handler for #DB (Debug exception).
11652 */
11653static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11654{
11655 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11656 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11657 Log6(("XcptDB\n"));
11658
11659 /*
11660 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11661 * for processing.
11662 */
11663 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11664 AssertRCReturn(rc, rc);
11665
11666 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11667 uint64_t uDR6 = X86_DR6_INIT_VAL;
11668 uDR6 |= ( pVmxTransient->uExitQualification
11669 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11670
11671 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11672 if (rc == VINF_EM_RAW_GUEST_TRAP)
11673 {
11674 /*
11675 * The exception was for the guest. Update DR6, DR7.GD and
11676 * IA32_DEBUGCTL.LBR before forwarding it.
11677 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11678 */
11679 VMMRZCallRing3Disable(pVCpu);
11680 HM_DISABLE_PREEMPT_IF_NEEDED();
11681
11682 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11683 pMixedCtx->dr[6] |= uDR6;
11684 if (CPUMIsGuestDebugStateActive(pVCpu))
11685 ASMSetDR6(pMixedCtx->dr[6]);
11686
11687 HM_RESTORE_PREEMPT_IF_NEEDED();
11688 VMMRZCallRing3Enable(pVCpu);
11689
11690 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11691 AssertRCReturn(rc, rc);
11692
11693 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11694 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11695
11696 /* Paranoia. */
11697 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11698 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11699
11700 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11701 AssertRCReturn(rc, rc);
11702
11703 /*
11704 * Raise #DB in the guest.
11705 *
11706 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11707 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11708 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11709 *
11710 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11711 */
11712 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11713 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11714 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11715 AssertRCReturn(rc, rc);
11716 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11717 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11718 return VINF_SUCCESS;
11719 }
11720
11721 /*
11722 * Not a guest trap, must be a hypervisor related debug event then.
11723 * Update DR6 in case someone is interested in it.
11724 */
11725 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11726 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11727 CPUMSetHyperDR6(pVCpu, uDR6);
11728
11729 return rc;
11730}
11731
11732
11733/**
11734 * VM-exit exception handler for #NM (Device-not-available exception: floating
11735 * point exception).
11736 */
11737static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11738{
11739 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11740
11741 /* We require CR0 and EFER. EFER is always up-to-date. */
11742 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11743 AssertRCReturn(rc, rc);
11744
11745 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11746 VMMRZCallRing3Disable(pVCpu);
11747 HM_DISABLE_PREEMPT_IF_NEEDED();
11748
11749 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11750 if (pVmxTransient->fWasGuestFPUStateActive)
11751 {
11752 rc = VINF_EM_RAW_GUEST_TRAP;
11753 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11754 }
11755 else
11756 {
11757#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11758 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11759#endif
11760 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11761 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11762 }
11763
11764 HM_RESTORE_PREEMPT_IF_NEEDED();
11765 VMMRZCallRing3Enable(pVCpu);
11766
11767 if (rc == VINF_SUCCESS)
11768 {
11769 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11770 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11771 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11772 pVCpu->hm.s.fUseGuestFpu = true;
11773 }
11774 else
11775 {
11776 /* Forward #NM to the guest. */
11777 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11778 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11779 AssertRCReturn(rc, rc);
11780 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11781 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11782 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11783 }
11784
11785 return VINF_SUCCESS;
11786}
11787
11788
11789/**
11790 * VM-exit exception handler for #GP (General-protection exception).
11791 *
11792 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11793 */
11794static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11795{
11796 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11797 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11798
11799 int rc = VERR_INTERNAL_ERROR_5;
11800 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11801 {
11802#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11803 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11804 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11805 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11806 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11807 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11808 AssertRCReturn(rc, rc);
11809 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11810 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11811 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11812 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11813 return rc;
11814#else
11815 /* We don't intercept #GP. */
11816 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11817 NOREF(pVmxTransient);
11818 return VERR_VMX_UNEXPECTED_EXCEPTION;
11819#endif
11820 }
11821
11822 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11823 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11824
11825 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11826 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11827 AssertRCReturn(rc, rc);
11828
11829 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11830 uint32_t cbOp = 0;
11831 PVM pVM = pVCpu->CTX_SUFF(pVM);
11832 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11833 if (RT_SUCCESS(rc))
11834 {
11835 rc = VINF_SUCCESS;
11836 Assert(cbOp == pDis->cbInstr);
11837 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11838 switch (pDis->pCurInstr->uOpcode)
11839 {
11840 case OP_CLI:
11841 {
11842 pMixedCtx->eflags.Bits.u1IF = 0;
11843 pMixedCtx->eflags.Bits.u1RF = 0;
11844 pMixedCtx->rip += pDis->cbInstr;
11845 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11846 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11847 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11848 break;
11849 }
11850
11851 case OP_STI:
11852 {
11853 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11854 pMixedCtx->eflags.Bits.u1IF = 1;
11855 pMixedCtx->eflags.Bits.u1RF = 0;
11856 pMixedCtx->rip += pDis->cbInstr;
11857 if (!fOldIF)
11858 {
11859 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11860 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11861 }
11862 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11863 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11864 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11865 break;
11866 }
11867
11868 case OP_HLT:
11869 {
11870 rc = VINF_EM_HALT;
11871 pMixedCtx->rip += pDis->cbInstr;
11872 pMixedCtx->eflags.Bits.u1RF = 0;
11873 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11874 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11875 break;
11876 }
11877
11878 case OP_POPF:
11879 {
11880 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11881 uint32_t cbParm;
11882 uint32_t uMask;
11883 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11884 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11885 {
11886 cbParm = 4;
11887 uMask = 0xffffffff;
11888 }
11889 else
11890 {
11891 cbParm = 2;
11892 uMask = 0xffff;
11893 }
11894
11895 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11896 RTGCPTR GCPtrStack = 0;
11897 X86EFLAGS Eflags;
11898 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11899 &GCPtrStack);
11900 if (RT_SUCCESS(rc))
11901 {
11902 Assert(sizeof(Eflags.u32) >= cbParm);
11903 Eflags.u32 = 0;
11904 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11905 }
11906 if (RT_FAILURE(rc))
11907 {
11908 rc = VERR_EM_INTERPRETER;
11909 break;
11910 }
11911 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11912 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
11913 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11914 pMixedCtx->esp += cbParm;
11915 pMixedCtx->esp &= uMask;
11916 pMixedCtx->rip += pDis->cbInstr;
11917 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11918 | HM_CHANGED_GUEST_RSP
11919 | HM_CHANGED_GUEST_RFLAGS);
11920 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11921 if (fStepping)
11922 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11923
11924 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11925 break;
11926 }
11927
11928 case OP_PUSHF:
11929 {
11930 uint32_t cbParm;
11931 uint32_t uMask;
11932 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11933 {
11934 cbParm = 4;
11935 uMask = 0xffffffff;
11936 }
11937 else
11938 {
11939 cbParm = 2;
11940 uMask = 0xffff;
11941 }
11942
11943 /* Get the stack pointer & push the contents of eflags onto the stack. */
11944 RTGCPTR GCPtrStack = 0;
11945 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11946 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11947 if (RT_FAILURE(rc))
11948 {
11949 rc = VERR_EM_INTERPRETER;
11950 break;
11951 }
11952 X86EFLAGS Eflags = pMixedCtx->eflags;
11953 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11954 Eflags.Bits.u1RF = 0;
11955 Eflags.Bits.u1VM = 0;
11956
11957 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11958 if (RT_FAILURE(rc))
11959 {
11960 rc = VERR_EM_INTERPRETER;
11961 break;
11962 }
11963 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11964 pMixedCtx->esp -= cbParm;
11965 pMixedCtx->esp &= uMask;
11966 pMixedCtx->rip += pDis->cbInstr;
11967 pMixedCtx->eflags.Bits.u1RF = 0;
11968 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11969 | HM_CHANGED_GUEST_RSP
11970 | HM_CHANGED_GUEST_RFLAGS);
11971 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11972 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11973 break;
11974 }
11975
11976 case OP_IRET:
11977 {
11978 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11979 * instruction reference. */
11980 RTGCPTR GCPtrStack = 0;
11981 uint32_t uMask = 0xffff;
11982 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11983 uint16_t aIretFrame[3];
11984 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11985 {
11986 rc = VERR_EM_INTERPRETER;
11987 break;
11988 }
11989 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11990 &GCPtrStack);
11991 if (RT_SUCCESS(rc))
11992 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11993 if (RT_FAILURE(rc))
11994 {
11995 rc = VERR_EM_INTERPRETER;
11996 break;
11997 }
11998 pMixedCtx->eip = 0;
11999 pMixedCtx->ip = aIretFrame[0];
12000 pMixedCtx->cs.Sel = aIretFrame[1];
12001 pMixedCtx->cs.ValidSel = aIretFrame[1];
12002 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
12003 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
12004 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
12005 pMixedCtx->sp += sizeof(aIretFrame);
12006 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12007 | HM_CHANGED_GUEST_SEGMENT_REGS
12008 | HM_CHANGED_GUEST_RSP
12009 | HM_CHANGED_GUEST_RFLAGS);
12010 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
12011 if (fStepping)
12012 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
12013 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
12014 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
12015 break;
12016 }
12017
12018 case OP_INT:
12019 {
12020 uint16_t uVector = pDis->Param1.uValue & 0xff;
12021 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
12022 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12023 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12024 break;
12025 }
12026
12027 case OP_INTO:
12028 {
12029 if (pMixedCtx->eflags.Bits.u1OF)
12030 {
12031 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
12032 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12033 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12034 }
12035 else
12036 {
12037 pMixedCtx->eflags.Bits.u1RF = 0;
12038 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12039 }
12040 break;
12041 }
12042
12043 default:
12044 {
12045 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
12046 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
12047 EMCODETYPE_SUPERVISOR);
12048 rc = VBOXSTRICTRC_VAL(rc2);
12049 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
12050 /** @todo We have to set pending-debug exceptions here when the guest is
12051 * single-stepping depending on the instruction that was interpreted. */
12052 Log4(("#GP rc=%Rrc\n", rc));
12053 break;
12054 }
12055 }
12056 }
12057 else
12058 rc = VERR_EM_INTERPRETER;
12059
12060 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
12061 ("#GP Unexpected rc=%Rrc\n", rc));
12062 return rc;
12063}
12064
12065
12066#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12067/**
12068 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
12069 * the exception reported in the VMX transient structure back into the VM.
12070 *
12071 * @remarks Requires uExitIntInfo in the VMX transient structure to be
12072 * up-to-date.
12073 */
12074static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12075{
12076 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12077
12078 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12079 hmR0VmxCheckExitDueToEventDelivery(). */
12080 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12081 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12082 AssertRCReturn(rc, rc);
12083 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12084
12085#ifdef DEBUG_ramshankar
12086 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12087 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12088 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12089#endif
12090
12091 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12092 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12093 return VINF_SUCCESS;
12094}
12095#endif
12096
12097
12098/**
12099 * VM-exit exception handler for #PF (Page-fault exception).
12100 */
12101static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12102{
12103 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12104 PVM pVM = pVCpu->CTX_SUFF(pVM);
12105 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12106 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12107 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12108 AssertRCReturn(rc, rc);
12109
12110#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12111 if (pVM->hm.s.fNestedPaging)
12112 {
12113 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12114 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12115 {
12116 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12117 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12118 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12119 }
12120 else
12121 {
12122 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12123 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12124 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12125 }
12126 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12127 return rc;
12128 }
12129#else
12130 Assert(!pVM->hm.s.fNestedPaging);
12131 NOREF(pVM);
12132#endif
12133
12134 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12135 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12136 if (pVmxTransient->fVectoringPF)
12137 {
12138 Assert(pVCpu->hm.s.Event.fPending);
12139 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12140 }
12141
12142 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12143 AssertRCReturn(rc, rc);
12144
12145 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12146 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12147
12148 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12149 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12150 (RTGCPTR)pVmxTransient->uExitQualification);
12151
12152 Log4(("#PF: rc=%Rrc\n", rc));
12153 if (rc == VINF_SUCCESS)
12154 {
12155 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12156 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12157 * memory? We don't update the whole state here... */
12158 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12159 | HM_CHANGED_GUEST_RSP
12160 | HM_CHANGED_GUEST_RFLAGS
12161 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12162 TRPMResetTrap(pVCpu);
12163 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12164 return rc;
12165 }
12166
12167 if (rc == VINF_EM_RAW_GUEST_TRAP)
12168 {
12169 if (!pVmxTransient->fVectoringDoublePF)
12170 {
12171 /* It's a guest page fault and needs to be reflected to the guest. */
12172 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12173 TRPMResetTrap(pVCpu);
12174 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12175 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12176 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12177 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12178 }
12179 else
12180 {
12181 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12182 TRPMResetTrap(pVCpu);
12183 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12184 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12185 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12186 }
12187
12188 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12189 return VINF_SUCCESS;
12190 }
12191
12192 TRPMResetTrap(pVCpu);
12193 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12194 return rc;
12195}
12196
12197/** @} */
12198
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette