VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 53631

Last change on this file since 53631 was 53631, checked in by vboxsync, 10 years ago

vmexit trace points.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 506.2 KB
Line 
1/* $Id: HMVMXR0.cpp 53631 2015-01-01 23:53:43Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2014 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#include <VBox/vmm/gim.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39# include "dtrace/VBoxVMM.h"
40
41#ifdef DEBUG_ramshankar
42# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
43# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
44# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
45# define HMVMX_ALWAYS_CHECK_GUEST_STATE
46# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
47# define HMVMX_ALWAYS_TRAP_PF
48# define HMVMX_ALWAYS_SWAP_FPU_STATE
49# define HMVMX_ALWAYS_FLUSH_TLB
50# define HMVMX_ALWAYS_SWAP_EFER
51#endif
52
53
54/*******************************************************************************
55* Defined Constants And Macros *
56*******************************************************************************/
57#if defined(RT_ARCH_AMD64)
58# define HMVMX_IS_64BIT_HOST_MODE() (true)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
61extern "C" uint32_t g_fVMXIs64bitHost;
62# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
63typedef uint64_t HMVMXHCUINTREG;
64#else
65# define HMVMX_IS_64BIT_HOST_MODE() (false)
66typedef RTHCUINTREG HMVMXHCUINTREG;
67#endif
68
69/** Use the function table. */
70#define HMVMX_USE_FUNCTION_TABLE
71
72/** Determine which tagged-TLB flush handler to use. */
73#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
74#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
75#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
76#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
77
78/** @name Updated-guest-state flags.
79 * @{ */
80#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
81#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
82#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
83#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
84#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
85#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
86#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
87#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
88#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
89#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
90#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
91#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
92#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
93#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
94#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
95#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
96#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
97#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
98#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
99#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
100#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
101 | HMVMX_UPDATED_GUEST_RSP \
102 | HMVMX_UPDATED_GUEST_RFLAGS \
103 | HMVMX_UPDATED_GUEST_CR0 \
104 | HMVMX_UPDATED_GUEST_CR3 \
105 | HMVMX_UPDATED_GUEST_CR4 \
106 | HMVMX_UPDATED_GUEST_GDTR \
107 | HMVMX_UPDATED_GUEST_IDTR \
108 | HMVMX_UPDATED_GUEST_LDTR \
109 | HMVMX_UPDATED_GUEST_TR \
110 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
111 | HMVMX_UPDATED_GUEST_DEBUG \
112 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
113 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
114 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
115 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
116 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
117 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
118 | HMVMX_UPDATED_GUEST_INTR_STATE \
119 | HMVMX_UPDATED_GUEST_APIC_STATE)
120/** @} */
121
122/** @name
123 * Flags to skip redundant reads of some common VMCS fields that are not part of
124 * the guest-CPU state but are in the transient structure.
125 */
126#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
127#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
130#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
131#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
132#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
133/** @} */
134
135/** @name
136 * States of the VMCS.
137 *
138 * This does not reflect all possible VMCS states but currently only those
139 * needed for maintaining the VMCS consistently even when thread-context hooks
140 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
141 */
142#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
143#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
144#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
145/** @} */
146
147/**
148 * Exception bitmap mask for real-mode guests (real-on-v86).
149 *
150 * We need to intercept all exceptions manually (except #PF). #NM is also
151 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
152 * even in real-mode if we have Nested Paging support.
153 */
154#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
155 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
156 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
157 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
158 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
159 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
160 | RT_BIT(X86_XCPT_XF))
161
162/**
163 * Exception bitmap mask for all contributory exceptions.
164 *
165 * Page fault is deliberately excluded here as it's conditional as to whether
166 * it's contributory or benign. Page faults are handled separately.
167 */
168#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
169 | RT_BIT(X86_XCPT_DE))
170
171/** Maximum VM-instruction error number. */
172#define HMVMX_INSTR_ERROR_MAX 28
173
174/** Profiling macro. */
175#ifdef HM_PROFILE_EXIT_DISPATCH
176# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
177# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
178#else
179# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
180# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
181#endif
182
183/** Assert that preemption is disabled or covered by thread-context hooks. */
184#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
185 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
186
187/** Assert that we haven't migrated CPUs when thread-context hooks are not
188 * used. */
189#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
190 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
191 ("Illegal migration! Entered on CPU %u Current %u\n", \
192 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
193
194/** Helper macro for VM-exit handlers called unexpectedly. */
195#define HMVMX_RETURN_UNEXPECTED_EXIT() \
196 do { \
197 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
198 return VERR_VMX_UNEXPECTED_EXIT; \
199 } while (0)
200
201
202/*******************************************************************************
203* Structures and Typedefs *
204*******************************************************************************/
205/**
206 * VMX transient state.
207 *
208 * A state structure for holding miscellaneous information across
209 * VMX non-root operation and restored after the transition.
210 */
211typedef struct VMXTRANSIENT
212{
213 /** The host's rflags/eflags. */
214 RTCCUINTREG uEflags;
215#if HC_ARCH_BITS == 32
216 uint32_t u32Alignment0;
217#endif
218 /** The guest's TPR value used for TPR shadowing. */
219 uint8_t u8GuestTpr;
220 /** Alignment. */
221 uint8_t abAlignment0[7];
222
223 /** The basic VM-exit reason. */
224 uint16_t uExitReason;
225 /** Alignment. */
226 uint16_t u16Alignment0;
227 /** The VM-exit interruption error code. */
228 uint32_t uExitIntErrorCode;
229 /** The VM-exit exit code qualification. */
230 uint64_t uExitQualification;
231
232 /** The VM-exit interruption-information field. */
233 uint32_t uExitIntInfo;
234 /** The VM-exit instruction-length field. */
235 uint32_t cbInstr;
236 /** The VM-exit instruction-information field. */
237 union
238 {
239 /** Plain unsigned int representation. */
240 uint32_t u;
241 /** INS and OUTS information. */
242 struct
243 {
244 uint32_t u6Reserved0 : 7;
245 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
246 uint32_t u3AddrSize : 3;
247 uint32_t u5Reserved1 : 5;
248 /** The segment register (X86_SREG_XXX). */
249 uint32_t iSegReg : 3;
250 uint32_t uReserved2 : 14;
251 } StrIo;
252 } ExitInstrInfo;
253 /** Whether the VM-entry failed or not. */
254 bool fVMEntryFailed;
255 /** Alignment. */
256 uint8_t abAlignment1[3];
257
258 /** The VM-entry interruption-information field. */
259 uint32_t uEntryIntInfo;
260 /** The VM-entry exception error code field. */
261 uint32_t uEntryXcptErrorCode;
262 /** The VM-entry instruction length field. */
263 uint32_t cbEntryInstr;
264
265 /** IDT-vectoring information field. */
266 uint32_t uIdtVectoringInfo;
267 /** IDT-vectoring error code. */
268 uint32_t uIdtVectoringErrorCode;
269
270 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
271 uint32_t fVmcsFieldsRead;
272
273 /** Whether the guest FPU was active at the time of VM-exit. */
274 bool fWasGuestFPUStateActive;
275 /** Whether the guest debug state was active at the time of VM-exit. */
276 bool fWasGuestDebugStateActive;
277 /** Whether the hyper debug state was active at the time of VM-exit. */
278 bool fWasHyperDebugStateActive;
279 /** Whether TSC-offsetting should be setup before VM-entry. */
280 bool fUpdateTscOffsettingAndPreemptTimer;
281 /** Whether the VM-exit was caused by a page-fault during delivery of a
282 * contributory exception or a page-fault. */
283 bool fVectoringDoublePF;
284 /** Whether the VM-exit was caused by a page-fault during delivery of an
285 * external interrupt or NMI. */
286 bool fVectoringPF;
287} VMXTRANSIENT;
288AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
289AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
290AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
291AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
292AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
293/** Pointer to VMX transient state. */
294typedef VMXTRANSIENT *PVMXTRANSIENT;
295
296
297/**
298 * MSR-bitmap read permissions.
299 */
300typedef enum VMXMSREXITREAD
301{
302 /** Reading this MSR causes a VM-exit. */
303 VMXMSREXIT_INTERCEPT_READ = 0xb,
304 /** Reading this MSR does not cause a VM-exit. */
305 VMXMSREXIT_PASSTHRU_READ
306} VMXMSREXITREAD;
307/** Pointer to MSR-bitmap read permissions. */
308typedef VMXMSREXITREAD* PVMXMSREXITREAD;
309
310/**
311 * MSR-bitmap write permissions.
312 */
313typedef enum VMXMSREXITWRITE
314{
315 /** Writing to this MSR causes a VM-exit. */
316 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
317 /** Writing to this MSR does not cause a VM-exit. */
318 VMXMSREXIT_PASSTHRU_WRITE
319} VMXMSREXITWRITE;
320/** Pointer to MSR-bitmap write permissions. */
321typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
322
323
324/**
325 * VMX VM-exit handler.
326 *
327 * @returns VBox status code.
328 * @param pVCpu Pointer to the VMCPU.
329 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
330 * out-of-sync. Make sure to update the required
331 * fields before using them.
332 * @param pVmxTransient Pointer to the VMX-transient structure.
333 */
334#ifndef HMVMX_USE_FUNCTION_TABLE
335typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
336#else
337typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
338/** Pointer to VM-exit handler. */
339typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
340#endif
341
342
343/*******************************************************************************
344* Internal Functions *
345*******************************************************************************/
346static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
347static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
348static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
349 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
350 bool fStepping, uint32_t *puIntState);
351#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
352static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
353#endif
354#ifndef HMVMX_USE_FUNCTION_TABLE
355DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
356# define HMVMX_EXIT_DECL static int
357#else
358# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
359#endif
360DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
361 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart);
362
363/** @name VM-exit handlers.
364 * @{
365 */
366static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
367static FNVMXEXITHANDLER hmR0VmxExitExtInt;
368static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
369static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
370static FNVMXEXITHANDLER hmR0VmxExitSipi;
371static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
372static FNVMXEXITHANDLER hmR0VmxExitSmi;
373static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
374static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
375static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
376static FNVMXEXITHANDLER hmR0VmxExitCpuid;
377static FNVMXEXITHANDLER hmR0VmxExitGetsec;
378static FNVMXEXITHANDLER hmR0VmxExitHlt;
379static FNVMXEXITHANDLER hmR0VmxExitInvd;
380static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
381static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
382static FNVMXEXITHANDLER hmR0VmxExitVmcall;
383static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
384static FNVMXEXITHANDLER hmR0VmxExitRsm;
385static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
386static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
387static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
388static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
389static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
390static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
391static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
392static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
393static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
394static FNVMXEXITHANDLER hmR0VmxExitMwait;
395static FNVMXEXITHANDLER hmR0VmxExitMtf;
396static FNVMXEXITHANDLER hmR0VmxExitMonitor;
397static FNVMXEXITHANDLER hmR0VmxExitPause;
398static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
399static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
400static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
401static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
402static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
403static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
404static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
405static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
406static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
407static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
408static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
409static FNVMXEXITHANDLER hmR0VmxExitRdrand;
410static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
411/** @} */
412
413static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
414static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
415static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
416static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
417static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
418static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
419#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
420static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
421#endif
422static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
423
424/*******************************************************************************
425* Global Variables *
426*******************************************************************************/
427#ifdef HMVMX_USE_FUNCTION_TABLE
428
429/**
430 * VMX_EXIT dispatch table.
431 */
432static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
433{
434 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
435 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
436 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
437 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
438 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
439 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
440 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
441 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
442 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
443 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
444 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
445 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
446 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
447 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
448 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
449 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
450 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
451 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
452 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
453 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
454 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
455 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
456 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
457 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
458 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
459 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
460 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
461 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
462 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
463 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
464 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
465 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
466 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
467 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
468 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
469 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
470 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
471 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
472 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
473 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
474 /* 40 UNDEFINED */ hmR0VmxExitPause,
475 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
476 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
477 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
478 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
479 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
480 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
481 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
482 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
483 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
484 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
485 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
486 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
487 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
488 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
489 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
490 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
491 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
492 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
493 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
494};
495#endif /* HMVMX_USE_FUNCTION_TABLE */
496
497#ifdef VBOX_STRICT
498static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
499{
500 /* 0 */ "(Not Used)",
501 /* 1 */ "VMCALL executed in VMX root operation.",
502 /* 2 */ "VMCLEAR with invalid physical address.",
503 /* 3 */ "VMCLEAR with VMXON pointer.",
504 /* 4 */ "VMLAUNCH with non-clear VMCS.",
505 /* 5 */ "VMRESUME with non-launched VMCS.",
506 /* 6 */ "VMRESUME after VMXOFF",
507 /* 7 */ "VM-entry with invalid control fields.",
508 /* 8 */ "VM-entry with invalid host state fields.",
509 /* 9 */ "VMPTRLD with invalid physical address.",
510 /* 10 */ "VMPTRLD with VMXON pointer.",
511 /* 11 */ "VMPTRLD with incorrect revision identifier.",
512 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
513 /* 13 */ "VMWRITE to read-only VMCS component.",
514 /* 14 */ "(Not Used)",
515 /* 15 */ "VMXON executed in VMX root operation.",
516 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
517 /* 17 */ "VM-entry with non-launched executing VMCS.",
518 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
519 /* 19 */ "VMCALL with non-clear VMCS.",
520 /* 20 */ "VMCALL with invalid VM-exit control fields.",
521 /* 21 */ "(Not Used)",
522 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
523 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
524 /* 24 */ "VMCALL with invalid SMM-monitor features.",
525 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
526 /* 26 */ "VM-entry with events blocked by MOV SS.",
527 /* 27 */ "(Not Used)",
528 /* 28 */ "Invalid operand to INVEPT/INVVPID."
529};
530#endif /* VBOX_STRICT */
531
532
533
534/**
535 * Updates the VM's last error record. If there was a VMX instruction error,
536 * reads the error data from the VMCS and updates VCPU's last error record as
537 * well.
538 *
539 * @param pVM Pointer to the VM.
540 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
541 * VERR_VMX_UNABLE_TO_START_VM or
542 * VERR_VMX_INVALID_VMCS_FIELD).
543 * @param rc The error code.
544 */
545static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
546{
547 AssertPtr(pVM);
548 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
549 || rc == VERR_VMX_UNABLE_TO_START_VM)
550 {
551 AssertPtrReturnVoid(pVCpu);
552 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
553 }
554 pVM->hm.s.lLastError = rc;
555}
556
557
558/**
559 * Reads the VM-entry interruption-information field from the VMCS into the VMX
560 * transient structure.
561 *
562 * @returns VBox status code.
563 * @param pVmxTransient Pointer to the VMX transient structure.
564 *
565 * @remarks No-long-jump zone!!!
566 */
567DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
568{
569 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
570 AssertRCReturn(rc, rc);
571 return VINF_SUCCESS;
572}
573
574
575/**
576 * Reads the VM-entry exception error code field from the VMCS into
577 * the VMX transient structure.
578 *
579 * @returns VBox status code.
580 * @param pVmxTransient Pointer to the VMX transient structure.
581 *
582 * @remarks No-long-jump zone!!!
583 */
584DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
585{
586 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
587 AssertRCReturn(rc, rc);
588 return VINF_SUCCESS;
589}
590
591
592/**
593 * Reads the VM-entry exception error code field from the VMCS into
594 * the VMX transient structure.
595 *
596 * @returns VBox status code.
597 * @param pVmxTransient Pointer to the VMX transient structure.
598 *
599 * @remarks No-long-jump zone!!!
600 */
601DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
602{
603 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
604 AssertRCReturn(rc, rc);
605 return VINF_SUCCESS;
606}
607
608
609/**
610 * Reads the VM-exit interruption-information field from the VMCS into the VMX
611 * transient structure.
612 *
613 * @returns VBox status code.
614 * @param pVmxTransient Pointer to the VMX transient structure.
615 */
616DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
617{
618 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
619 {
620 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
621 AssertRCReturn(rc, rc);
622 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
623 }
624 return VINF_SUCCESS;
625}
626
627
628/**
629 * Reads the VM-exit interruption error code from the VMCS into the VMX
630 * transient structure.
631 *
632 * @returns VBox status code.
633 * @param pVmxTransient Pointer to the VMX transient structure.
634 */
635DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
636{
637 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
638 {
639 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
640 AssertRCReturn(rc, rc);
641 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
642 }
643 return VINF_SUCCESS;
644}
645
646
647/**
648 * Reads the VM-exit instruction length field from the VMCS into the VMX
649 * transient structure.
650 *
651 * @returns VBox status code.
652 * @param pVCpu Pointer to the VMCPU.
653 * @param pVmxTransient Pointer to the VMX transient structure.
654 */
655DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
656{
657 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
658 {
659 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
660 AssertRCReturn(rc, rc);
661 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
662 }
663 return VINF_SUCCESS;
664}
665
666
667/**
668 * Reads the VM-exit instruction-information field from the VMCS into
669 * the VMX transient structure.
670 *
671 * @returns VBox status code.
672 * @param pVmxTransient Pointer to the VMX transient structure.
673 */
674DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
675{
676 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
677 {
678 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
679 AssertRCReturn(rc, rc);
680 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
681 }
682 return VINF_SUCCESS;
683}
684
685
686/**
687 * Reads the exit code qualification from the VMCS into the VMX transient
688 * structure.
689 *
690 * @returns VBox status code.
691 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
692 * case).
693 * @param pVmxTransient Pointer to the VMX transient structure.
694 */
695DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
696{
697 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
698 {
699 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
700 AssertRCReturn(rc, rc);
701 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
702 }
703 return VINF_SUCCESS;
704}
705
706
707/**
708 * Reads the IDT-vectoring information field from the VMCS into the VMX
709 * transient structure.
710 *
711 * @returns VBox status code.
712 * @param pVmxTransient Pointer to the VMX transient structure.
713 *
714 * @remarks No-long-jump zone!!!
715 */
716DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
717{
718 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
719 {
720 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
721 AssertRCReturn(rc, rc);
722 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
723 }
724 return VINF_SUCCESS;
725}
726
727
728/**
729 * Reads the IDT-vectoring error code from the VMCS into the VMX
730 * transient structure.
731 *
732 * @returns VBox status code.
733 * @param pVmxTransient Pointer to the VMX transient structure.
734 */
735DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
736{
737 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
738 {
739 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
740 AssertRCReturn(rc, rc);
741 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
742 }
743 return VINF_SUCCESS;
744}
745
746
747/**
748 * Enters VMX root mode operation on the current CPU.
749 *
750 * @returns VBox status code.
751 * @param pVM Pointer to the VM (optional, can be NULL, after
752 * a resume).
753 * @param HCPhysCpuPage Physical address of the VMXON region.
754 * @param pvCpuPage Pointer to the VMXON region.
755 */
756static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
757{
758 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
759 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
760 Assert(pvCpuPage);
761 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
762
763 if (pVM)
764 {
765 /* Write the VMCS revision dword to the VMXON region. */
766 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
767 }
768
769 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
770 RTCCUINTREG uEflags = ASMIntDisableFlags();
771
772 /* Enable the VMX bit in CR4 if necessary. */
773 RTCCUINTREG uCr4 = ASMGetCR4();
774 if (!(uCr4 & X86_CR4_VMXE))
775 ASMSetCR4(uCr4 | X86_CR4_VMXE);
776
777 /* Enter VMX root mode. */
778 int rc = VMXEnable(HCPhysCpuPage);
779 if (RT_FAILURE(rc))
780 ASMSetCR4(uCr4);
781
782 /* Restore interrupts. */
783 ASMSetFlags(uEflags);
784 return rc;
785}
786
787
788/**
789 * Exits VMX root mode operation on the current CPU.
790 *
791 * @returns VBox status code.
792 */
793static int hmR0VmxLeaveRootMode(void)
794{
795 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
796
797 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
798 RTCCUINTREG uEflags = ASMIntDisableFlags();
799
800 /* If we're for some reason not in VMX root mode, then don't leave it. */
801 RTCCUINTREG uHostCR4 = ASMGetCR4();
802
803 int rc;
804 if (uHostCR4 & X86_CR4_VMXE)
805 {
806 /* Exit VMX root mode and clear the VMX bit in CR4. */
807 VMXDisable();
808 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
809 rc = VINF_SUCCESS;
810 }
811 else
812 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
813
814 /* Restore interrupts. */
815 ASMSetFlags(uEflags);
816 return rc;
817}
818
819
820/**
821 * Allocates and maps one physically contiguous page. The allocated page is
822 * zero'd out. (Used by various VT-x structures).
823 *
824 * @returns IPRT status code.
825 * @param pMemObj Pointer to the ring-0 memory object.
826 * @param ppVirt Where to store the virtual address of the
827 * allocation.
828 * @param pPhys Where to store the physical address of the
829 * allocation.
830 */
831DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
832{
833 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
834 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
835 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
836
837 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
838 if (RT_FAILURE(rc))
839 return rc;
840 *ppVirt = RTR0MemObjAddress(*pMemObj);
841 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
842 ASMMemZero32(*ppVirt, PAGE_SIZE);
843 return VINF_SUCCESS;
844}
845
846
847/**
848 * Frees and unmaps an allocated physical page.
849 *
850 * @param pMemObj Pointer to the ring-0 memory object.
851 * @param ppVirt Where to re-initialize the virtual address of
852 * allocation as 0.
853 * @param pHCPhys Where to re-initialize the physical address of the
854 * allocation as 0.
855 */
856DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
857{
858 AssertPtr(pMemObj);
859 AssertPtr(ppVirt);
860 AssertPtr(pHCPhys);
861 if (*pMemObj != NIL_RTR0MEMOBJ)
862 {
863 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
864 AssertRC(rc);
865 *pMemObj = NIL_RTR0MEMOBJ;
866 *ppVirt = 0;
867 *pHCPhys = 0;
868 }
869}
870
871
872/**
873 * Worker function to free VT-x related structures.
874 *
875 * @returns IPRT status code.
876 * @param pVM Pointer to the VM.
877 */
878static void hmR0VmxStructsFree(PVM pVM)
879{
880 for (VMCPUID i = 0; i < pVM->cCpus; i++)
881 {
882 PVMCPU pVCpu = &pVM->aCpus[i];
883 AssertPtr(pVCpu);
884
885 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
886 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
887
888 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
889 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
890
891 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
892 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
893 }
894
895 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
896#ifdef VBOX_WITH_CRASHDUMP_MAGIC
897 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
898#endif
899}
900
901
902/**
903 * Worker function to allocate VT-x related VM structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM Pointer to the VM.
907 */
908static int hmR0VmxStructsAlloc(PVM pVM)
909{
910 /*
911 * Initialize members up-front so we can cleanup properly on allocation failure.
912 */
913#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
914 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
915 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
916 pVM->hm.s.vmx.HCPhys##a_Name = 0;
917
918#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
919 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
920 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
921 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
922
923#ifdef VBOX_WITH_CRASHDUMP_MAGIC
924 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
925#endif
926 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
927
928 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
929 for (VMCPUID i = 0; i < pVM->cCpus; i++)
930 {
931 PVMCPU pVCpu = &pVM->aCpus[i];
932 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
933 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
934 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
935 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
936 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
937 }
938#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
939#undef VMXLOCAL_INIT_VM_MEMOBJ
940
941 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
942 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
943 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
944 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
945
946 /*
947 * Allocate all the VT-x structures.
948 */
949 int rc = VINF_SUCCESS;
950#ifdef VBOX_WITH_CRASHDUMP_MAGIC
951 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
952 if (RT_FAILURE(rc))
953 goto cleanup;
954 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
955 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
956#endif
957
958 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
959 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
960 {
961 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
962 &pVM->hm.s.vmx.HCPhysApicAccess);
963 if (RT_FAILURE(rc))
964 goto cleanup;
965 }
966
967 /*
968 * Initialize per-VCPU VT-x structures.
969 */
970 for (VMCPUID i = 0; i < pVM->cCpus; i++)
971 {
972 PVMCPU pVCpu = &pVM->aCpus[i];
973 AssertPtr(pVCpu);
974
975 /* Allocate the VM control structure (VMCS). */
976 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
977 if (RT_FAILURE(rc))
978 goto cleanup;
979
980 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
981 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
982 {
983 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
984 &pVCpu->hm.s.vmx.HCPhysVirtApic);
985 if (RT_FAILURE(rc))
986 goto cleanup;
987 }
988
989 /*
990 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
991 * transparent accesses of specific MSRs.
992 *
993 * If the condition for enabling MSR bitmaps changes here, don't forget to
994 * update HMIsMsrBitmapsAvailable().
995 */
996 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
997 {
998 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
999 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1000 if (RT_FAILURE(rc))
1001 goto cleanup;
1002 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1003 }
1004
1005 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1006 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1007 if (RT_FAILURE(rc))
1008 goto cleanup;
1009
1010 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1012 if (RT_FAILURE(rc))
1013 goto cleanup;
1014 }
1015
1016 return VINF_SUCCESS;
1017
1018cleanup:
1019 hmR0VmxStructsFree(pVM);
1020 return rc;
1021}
1022
1023
1024/**
1025 * Does global VT-x initialization (called during module initialization).
1026 *
1027 * @returns VBox status code.
1028 */
1029VMMR0DECL(int) VMXR0GlobalInit(void)
1030{
1031#ifdef HMVMX_USE_FUNCTION_TABLE
1032 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1033# ifdef VBOX_STRICT
1034 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1035 Assert(g_apfnVMExitHandlers[i]);
1036# endif
1037#endif
1038 return VINF_SUCCESS;
1039}
1040
1041
1042/**
1043 * Does global VT-x termination (called during module termination).
1044 */
1045VMMR0DECL(void) VMXR0GlobalTerm()
1046{
1047 /* Nothing to do currently. */
1048}
1049
1050
1051/**
1052 * Sets up and activates VT-x on the current CPU.
1053 *
1054 * @returns VBox status code.
1055 * @param pCpu Pointer to the global CPU info struct.
1056 * @param pVM Pointer to the VM (can be NULL after a host resume
1057 * operation).
1058 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1059 * fEnabledByHost is true).
1060 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1061 * @a fEnabledByHost is true).
1062 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1063 * enable VT-x on the host.
1064 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1065 */
1066VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1067 void *pvMsrs)
1068{
1069 Assert(pCpu);
1070 Assert(pvMsrs);
1071 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1072
1073 /* Enable VT-x if it's not already enabled by the host. */
1074 if (!fEnabledByHost)
1075 {
1076 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1077 if (RT_FAILURE(rc))
1078 return rc;
1079 }
1080
1081 /*
1082 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1083 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1084 */
1085 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1086 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1087 {
1088 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1089 pCpu->fFlushAsidBeforeUse = false;
1090 }
1091 else
1092 pCpu->fFlushAsidBeforeUse = true;
1093
1094 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1095 ++pCpu->cTlbFlushes;
1096
1097 return VINF_SUCCESS;
1098}
1099
1100
1101/**
1102 * Deactivates VT-x on the current CPU.
1103 *
1104 * @returns VBox status code.
1105 * @param pCpu Pointer to the global CPU info struct.
1106 * @param pvCpuPage Pointer to the VMXON region.
1107 * @param HCPhysCpuPage Physical address of the VMXON region.
1108 *
1109 * @remarks This function should never be called when SUPR0EnableVTx() or
1110 * similar was used to enable VT-x on the host.
1111 */
1112VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1113{
1114 NOREF(pCpu);
1115 NOREF(pvCpuPage);
1116 NOREF(HCPhysCpuPage);
1117
1118 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1119 return hmR0VmxLeaveRootMode();
1120}
1121
1122
1123/**
1124 * Sets the permission bits for the specified MSR in the MSR bitmap.
1125 *
1126 * @param pVCpu Pointer to the VMCPU.
1127 * @param uMSR The MSR value.
1128 * @param enmRead Whether reading this MSR causes a VM-exit.
1129 * @param enmWrite Whether writing this MSR causes a VM-exit.
1130 */
1131static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1132{
1133 int32_t iBit;
1134 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1135
1136 /*
1137 * Layout:
1138 * 0x000 - 0x3ff - Low MSR read bits
1139 * 0x400 - 0x7ff - High MSR read bits
1140 * 0x800 - 0xbff - Low MSR write bits
1141 * 0xc00 - 0xfff - High MSR write bits
1142 */
1143 if (uMsr <= 0x00001FFF)
1144 iBit = uMsr;
1145 else if ( uMsr >= 0xC0000000
1146 && uMsr <= 0xC0001FFF)
1147 {
1148 iBit = (uMsr - 0xC0000000);
1149 pbMsrBitmap += 0x400;
1150 }
1151 else
1152 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1153
1154 Assert(iBit <= 0x1fff);
1155 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1156 ASMBitSet(pbMsrBitmap, iBit);
1157 else
1158 ASMBitClear(pbMsrBitmap, iBit);
1159
1160 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1161 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1162 else
1163 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1164}
1165
1166
1167#ifdef VBOX_STRICT
1168/**
1169 * Gets the permission bits for the specified MSR in the MSR bitmap.
1170 *
1171 * @returns VBox status code.
1172 * @retval VINF_SUCCESS if the specified MSR is found.
1173 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1174 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1175 *
1176 * @param pVCpu Pointer to the VMCPU.
1177 * @param uMsr The MSR.
1178 * @param penmRead Where to store the read permissions.
1179 * @param penmWrite Where to store the write permissions.
1180 */
1181static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1182{
1183 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1184 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1185 int32_t iBit;
1186 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1187
1188 /* See hmR0VmxSetMsrPermission() for the layout. */
1189 if (uMsr <= 0x00001FFF)
1190 iBit = uMsr;
1191 else if ( uMsr >= 0xC0000000
1192 && uMsr <= 0xC0001FFF)
1193 {
1194 iBit = (uMsr - 0xC0000000);
1195 pbMsrBitmap += 0x400;
1196 }
1197 else
1198 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1199
1200 Assert(iBit <= 0x1fff);
1201 if (ASMBitTest(pbMsrBitmap, iBit))
1202 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1203 else
1204 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1205
1206 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1207 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1208 else
1209 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1210 return VINF_SUCCESS;
1211}
1212#endif /* VBOX_STRICT */
1213
1214
1215/**
1216 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1217 * area.
1218 *
1219 * @returns VBox status code.
1220 * @param pVCpu Pointer to the VMCPU.
1221 * @param cMsrs The number of MSRs.
1222 */
1223DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1224{
1225 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1226 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1227 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1228 {
1229 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1230 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1231 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1232 }
1233
1234 /* Update number of guest MSRs to load/store across the world-switch. */
1235 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1236 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1237
1238 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1239 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1240
1241 /* Update the VCPU's copy of the MSR count. */
1242 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1243
1244 return VINF_SUCCESS;
1245}
1246
1247
1248/**
1249 * Adds a new (or updates the value of an existing) guest/host MSR
1250 * pair to be swapped during the world-switch as part of the
1251 * auto-load/store MSR area in the VMCS.
1252 *
1253 * @returns true if the MSR was added -and- its value was updated, false
1254 * otherwise.
1255 * @param pVCpu Pointer to the VMCPU.
1256 * @param uMsr The MSR.
1257 * @param uGuestMsr Value of the guest MSR.
1258 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1259 * necessary.
1260 */
1261static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1262{
1263 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1264 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1265 uint32_t i;
1266 for (i = 0; i < cMsrs; i++)
1267 {
1268 if (pGuestMsr->u32Msr == uMsr)
1269 break;
1270 pGuestMsr++;
1271 }
1272
1273 bool fAdded = false;
1274 if (i == cMsrs)
1275 {
1276 ++cMsrs;
1277 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1278 AssertRC(rc);
1279
1280 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1281 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1282 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1283
1284 fAdded = true;
1285 }
1286
1287 /* Update the MSR values in the auto-load/store MSR area. */
1288 pGuestMsr->u32Msr = uMsr;
1289 pGuestMsr->u64Value = uGuestMsrValue;
1290
1291 /* Create/update the MSR slot in the host MSR area. */
1292 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1293 pHostMsr += i;
1294 pHostMsr->u32Msr = uMsr;
1295
1296 /*
1297 * Update the host MSR only when requested by the caller AND when we're
1298 * adding it to the auto-load/store area. Otherwise, it would have been
1299 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1300 */
1301 bool fUpdatedMsrValue = false;
1302 if ( fAdded
1303 && fUpdateHostMsr)
1304 {
1305 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1306 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1307 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1308 fUpdatedMsrValue = true;
1309 }
1310
1311 return fUpdatedMsrValue;
1312}
1313
1314
1315/**
1316 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1317 * auto-load/store MSR area in the VMCS.
1318 *
1319 * @returns VBox status code.
1320 * @param pVCpu Pointer to the VMCPU.
1321 * @param uMsr The MSR.
1322 */
1323static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1324{
1325 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1326 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1327 for (uint32_t i = 0; i < cMsrs; i++)
1328 {
1329 /* Find the MSR. */
1330 if (pGuestMsr->u32Msr == uMsr)
1331 {
1332 /* If it's the last MSR, simply reduce the count. */
1333 if (i == cMsrs - 1)
1334 {
1335 --cMsrs;
1336 break;
1337 }
1338
1339 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1340 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1341 pLastGuestMsr += cMsrs - 1;
1342 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1343 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1344
1345 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1346 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1347 pLastHostMsr += cMsrs - 1;
1348 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1349 pHostMsr->u64Value = pLastHostMsr->u64Value;
1350 --cMsrs;
1351 break;
1352 }
1353 pGuestMsr++;
1354 }
1355
1356 /* Update the VMCS if the count changed (meaning the MSR was found). */
1357 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1358 {
1359 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1360 AssertRCReturn(rc, rc);
1361
1362 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1363 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1364 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1365
1366 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1367 return VINF_SUCCESS;
1368 }
1369
1370 return VERR_NOT_FOUND;
1371}
1372
1373
1374/**
1375 * Checks if the specified guest MSR is part of the auto-load/store area in
1376 * the VMCS.
1377 *
1378 * @returns true if found, false otherwise.
1379 * @param pVCpu Pointer to the VMCPU.
1380 * @param uMsr The MSR to find.
1381 */
1382static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1383{
1384 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1385 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1386
1387 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1388 {
1389 if (pGuestMsr->u32Msr == uMsr)
1390 return true;
1391 }
1392 return false;
1393}
1394
1395
1396/**
1397 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1398 *
1399 * @param pVCpu Pointer to the VMCPU.
1400 *
1401 * @remarks No-long-jump zone!!!
1402 */
1403static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1404{
1405 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1406 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1407 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1408 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1409
1410 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1411 {
1412 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1413
1414 /*
1415 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1416 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1417 */
1418 if (pHostMsr->u32Msr == MSR_K6_EFER)
1419 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1420 else
1421 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1422 }
1423
1424 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1425}
1426
1427
1428#if HC_ARCH_BITS == 64
1429/**
1430 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1431 * perform lazy restoration of the host MSRs while leaving VT-x.
1432 *
1433 * @param pVCpu Pointer to the VMCPU.
1434 *
1435 * @remarks No-long-jump zone!!!
1436 */
1437static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1438{
1439 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1440
1441 /*
1442 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1443 */
1444 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1445 {
1446 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1447 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1448 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1449 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1450 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1451 }
1452}
1453
1454
1455/**
1456 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1457 * lazily while leaving VT-x.
1458 *
1459 * @returns true if it does, false otherwise.
1460 * @param pVCpu Pointer to the VMCPU.
1461 * @param uMsr The MSR to check.
1462 */
1463static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1464{
1465 NOREF(pVCpu);
1466 switch (uMsr)
1467 {
1468 case MSR_K8_LSTAR:
1469 case MSR_K6_STAR:
1470 case MSR_K8_SF_MASK:
1471 case MSR_K8_KERNEL_GS_BASE:
1472 return true;
1473 }
1474 return false;
1475}
1476
1477
1478/**
1479 * Saves a set of guest MSRs back into the guest-CPU context.
1480 *
1481 * @param pVCpu Pointer to the VMCPU.
1482 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1483 * out-of-sync. Make sure to update the required fields
1484 * before using them.
1485 *
1486 * @remarks No-long-jump zone!!!
1487 */
1488static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1489{
1490 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1491 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1492
1493 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1494 {
1495 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1496 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1497 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1498 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1499 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1500 }
1501}
1502
1503
1504/**
1505 * Loads a set of guests MSRs to allow read/passthru to the guest.
1506 *
1507 * The name of this function is slightly confusing. This function does NOT
1508 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1509 * common prefix for functions dealing with "lazy restoration" of the shared
1510 * MSRs.
1511 *
1512 * @param pVCpu Pointer to the VMCPU.
1513 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1514 * out-of-sync. Make sure to update the required fields
1515 * before using them.
1516 *
1517 * @remarks No-long-jump zone!!!
1518 */
1519static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1520{
1521 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1522 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1523
1524#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1525 do { \
1526 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1527 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1528 else \
1529 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1530 } while (0)
1531
1532 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1533 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1534 {
1535 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1536 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1537 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1538 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1539 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1540 }
1541 else
1542 {
1543 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1544 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1545 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1546 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1547 }
1548
1549#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1550}
1551
1552
1553/**
1554 * Performs lazy restoration of the set of host MSRs if they were previously
1555 * loaded with guest MSR values.
1556 *
1557 * @param pVCpu Pointer to the VMCPU.
1558 *
1559 * @remarks No-long-jump zone!!!
1560 * @remarks The guest MSRs should have been saved back into the guest-CPU
1561 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1562 */
1563static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1564{
1565 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1566 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1567
1568 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1569 {
1570 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1571 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1572 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1573 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1574 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1575 }
1576 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1577}
1578#endif /* HC_ARCH_BITS == 64 */
1579
1580
1581/**
1582 * Verifies that our cached values of the VMCS controls are all
1583 * consistent with what's actually present in the VMCS.
1584 *
1585 * @returns VBox status code.
1586 * @param pVCpu Pointer to the VMCPU.
1587 */
1588static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1589{
1590 uint32_t u32Val;
1591 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1592 AssertRCReturn(rc, rc);
1593 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1594 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1595
1596 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1597 AssertRCReturn(rc, rc);
1598 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1599 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1600
1601 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1602 AssertRCReturn(rc, rc);
1603 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1604 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1605
1606 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1607 AssertRCReturn(rc, rc);
1608 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1609 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1610
1611 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1612 {
1613 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1614 AssertRCReturn(rc, rc);
1615 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1616 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1617 }
1618
1619 return VINF_SUCCESS;
1620}
1621
1622
1623#ifdef VBOX_STRICT
1624/**
1625 * Verifies that our cached host EFER value has not changed
1626 * since we cached it.
1627 *
1628 * @param pVCpu Pointer to the VMCPU.
1629 */
1630static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1631{
1632 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1633
1634 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1635 {
1636 uint64_t u64Val;
1637 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1638 AssertRC(rc);
1639
1640 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1641 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1642 }
1643}
1644
1645
1646/**
1647 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1648 * VMCS are correct.
1649 *
1650 * @param pVCpu Pointer to the VMCPU.
1651 */
1652static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1653{
1654 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1655
1656 /* Verify MSR counts in the VMCS are what we think it should be. */
1657 uint32_t cMsrs;
1658 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1659 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1660
1661 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1662 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1663
1664 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1665 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1666
1667 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1668 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1669 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1670 {
1671 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1672 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1673 pGuestMsr->u32Msr, cMsrs));
1674
1675 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1676 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1677 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1678
1679 /* Verify that the permissions are as expected in the MSR bitmap. */
1680 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1681 {
1682 VMXMSREXITREAD enmRead;
1683 VMXMSREXITWRITE enmWrite;
1684 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1685 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1686 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1687 {
1688 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1689 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1690 }
1691 else
1692 {
1693 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1694 pGuestMsr->u32Msr, cMsrs));
1695 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1696 pGuestMsr->u32Msr, cMsrs));
1697 }
1698 }
1699 }
1700}
1701#endif /* VBOX_STRICT */
1702
1703
1704/**
1705 * Flushes the TLB using EPT.
1706 *
1707 * @returns VBox status code.
1708 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1709 * enmFlush).
1710 * @param enmFlush Type of flush.
1711 *
1712 * @remarks Caller is responsible for making sure this function is called only
1713 * when NestedPaging is supported and providing @a enmFlush that is
1714 * supported by the CPU.
1715 * @remarks Can be called with interrupts disabled.
1716 */
1717static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1718{
1719 uint64_t au64Descriptor[2];
1720 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1721 au64Descriptor[0] = 0;
1722 else
1723 {
1724 Assert(pVCpu);
1725 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1726 }
1727 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1728
1729 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1730 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1731 rc));
1732 if ( RT_SUCCESS(rc)
1733 && pVCpu)
1734 {
1735 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1736 }
1737}
1738
1739
1740/**
1741 * Flushes the TLB using VPID.
1742 *
1743 * @returns VBox status code.
1744 * @param pVM Pointer to the VM.
1745 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1746 * enmFlush).
1747 * @param enmFlush Type of flush.
1748 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1749 * on @a enmFlush).
1750 *
1751 * @remarks Can be called with interrupts disabled.
1752 */
1753static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1754{
1755 NOREF(pVM);
1756 AssertPtr(pVM);
1757 Assert(pVM->hm.s.vmx.fVpid);
1758
1759 uint64_t au64Descriptor[2];
1760 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1761 {
1762 au64Descriptor[0] = 0;
1763 au64Descriptor[1] = 0;
1764 }
1765 else
1766 {
1767 AssertPtr(pVCpu);
1768 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1769 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1770 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1771 au64Descriptor[1] = GCPtr;
1772 }
1773
1774 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1775 AssertMsg(rc == VINF_SUCCESS,
1776 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1777 if ( RT_SUCCESS(rc)
1778 && pVCpu)
1779 {
1780 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1781 }
1782}
1783
1784
1785/**
1786 * Invalidates a guest page by guest virtual address. Only relevant for
1787 * EPT/VPID, otherwise there is nothing really to invalidate.
1788 *
1789 * @returns VBox status code.
1790 * @param pVM Pointer to the VM.
1791 * @param pVCpu Pointer to the VMCPU.
1792 * @param GCVirt Guest virtual address of the page to invalidate.
1793 */
1794VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1795{
1796 AssertPtr(pVM);
1797 AssertPtr(pVCpu);
1798 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1799
1800 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1801 if (!fFlushPending)
1802 {
1803 /*
1804 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1805 * See @bugref{6043} and @bugref{6177}.
1806 *
1807 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1808 * function maybe called in a loop with individual addresses.
1809 */
1810 if (pVM->hm.s.vmx.fVpid)
1811 {
1812 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1813 {
1814 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1815 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1816 }
1817 else
1818 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1819 }
1820 else if (pVM->hm.s.fNestedPaging)
1821 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1822 }
1823
1824 return VINF_SUCCESS;
1825}
1826
1827
1828/**
1829 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1830 * otherwise there is nothing really to invalidate.
1831 *
1832 * @returns VBox status code.
1833 * @param pVM Pointer to the VM.
1834 * @param pVCpu Pointer to the VMCPU.
1835 * @param GCPhys Guest physical address of the page to invalidate.
1836 */
1837VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1838{
1839 NOREF(pVM); NOREF(GCPhys);
1840 LogFlowFunc(("%RGp\n", GCPhys));
1841
1842 /*
1843 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1844 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1845 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1846 */
1847 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1848 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1849 return VINF_SUCCESS;
1850}
1851
1852
1853/**
1854 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1855 * case where neither EPT nor VPID is supported by the CPU.
1856 *
1857 * @param pVM Pointer to the VM.
1858 * @param pVCpu Pointer to the VMCPU.
1859 * @param pCpu Pointer to the global HM struct.
1860 *
1861 * @remarks Called with interrupts disabled.
1862 */
1863static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1864{
1865 AssertPtr(pVCpu);
1866 AssertPtr(pCpu);
1867 NOREF(pVM);
1868
1869 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1870
1871 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1872#if 0
1873 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1874 pVCpu->hm.s.TlbShootdown.cPages = 0;
1875#endif
1876
1877 Assert(pCpu->idCpu != NIL_RTCPUID);
1878 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1879 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1880 pVCpu->hm.s.fForceTLBFlush = false;
1881 return;
1882}
1883
1884
1885/**
1886 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1887 *
1888 * @param pVM Pointer to the VM.
1889 * @param pVCpu Pointer to the VMCPU.
1890 * @param pCpu Pointer to the global HM CPU struct.
1891 * @remarks All references to "ASID" in this function pertains to "VPID" in
1892 * Intel's nomenclature. The reason is, to avoid confusion in compare
1893 * statements since the host-CPU copies are named "ASID".
1894 *
1895 * @remarks Called with interrupts disabled.
1896 */
1897static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1898{
1899#ifdef VBOX_WITH_STATISTICS
1900 bool fTlbFlushed = false;
1901# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1902# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1903 if (!fTlbFlushed) \
1904 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1905 } while (0)
1906#else
1907# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1908# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1909#endif
1910
1911 AssertPtr(pVM);
1912 AssertPtr(pCpu);
1913 AssertPtr(pVCpu);
1914 Assert(pCpu->idCpu != NIL_RTCPUID);
1915
1916 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1917 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1918 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1919
1920 /*
1921 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1922 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1923 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1924 */
1925 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1926 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1927 {
1928 ++pCpu->uCurrentAsid;
1929 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1930 {
1931 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1932 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1933 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1934 }
1935
1936 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1937 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1938 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1939
1940 /*
1941 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1942 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1943 */
1944 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1945 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1946 HMVMX_SET_TAGGED_TLB_FLUSHED();
1947 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1948 }
1949
1950 /* Check for explicit TLB shootdowns. */
1951 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1952 {
1953 /*
1954 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1955 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1956 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1957 * but not guest-physical mappings.
1958 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1959 */
1960 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1961 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1962 HMVMX_SET_TAGGED_TLB_FLUSHED();
1963 }
1964
1965 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1966 * where it is commented out. Support individual entry flushing
1967 * someday. */
1968#if 0
1969 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1970 {
1971 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1972
1973 /*
1974 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1975 * as supported by the CPU.
1976 */
1977 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1978 {
1979 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1980 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1981 }
1982 else
1983 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1984
1985 HMVMX_SET_TAGGED_TLB_FLUSHED();
1986 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1987 pVCpu->hm.s.TlbShootdown.cPages = 0;
1988 }
1989#endif
1990
1991 pVCpu->hm.s.fForceTLBFlush = false;
1992
1993 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1994
1995 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1996 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1997 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1998 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1999 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2000 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2001 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2002 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2003 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2004
2005 /* Update VMCS with the VPID. */
2006 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2007 AssertRC(rc);
2008
2009#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2010}
2011
2012
2013/**
2014 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2015 *
2016 * @returns VBox status code.
2017 * @param pVM Pointer to the VM.
2018 * @param pVCpu Pointer to the VMCPU.
2019 * @param pCpu Pointer to the global HM CPU struct.
2020 *
2021 * @remarks Called with interrupts disabled.
2022 */
2023static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2024{
2025 AssertPtr(pVM);
2026 AssertPtr(pVCpu);
2027 AssertPtr(pCpu);
2028 Assert(pCpu->idCpu != NIL_RTCPUID);
2029 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2030 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2031
2032 /*
2033 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2034 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2035 */
2036 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2037 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2038 {
2039 pVCpu->hm.s.fForceTLBFlush = true;
2040 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2041 }
2042
2043 /* Check for explicit TLB shootdown flushes. */
2044 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2045 {
2046 pVCpu->hm.s.fForceTLBFlush = true;
2047 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2048 }
2049
2050 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2051 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2052
2053 if (pVCpu->hm.s.fForceTLBFlush)
2054 {
2055 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2056 pVCpu->hm.s.fForceTLBFlush = false;
2057 }
2058 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2059 * where it is commented out. Support individual entry flushing
2060 * someday. */
2061#if 0
2062 else
2063 {
2064 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2065 {
2066 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2067 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2068 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2069 }
2070 else
2071 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2072
2073 pVCpu->hm.s.TlbShootdown.cPages = 0;
2074 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2075 }
2076#endif
2077}
2078
2079
2080/**
2081 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2082 *
2083 * @returns VBox status code.
2084 * @param pVM Pointer to the VM.
2085 * @param pVCpu Pointer to the VMCPU.
2086 * @param pCpu Pointer to the global HM CPU struct.
2087 *
2088 * @remarks Called with interrupts disabled.
2089 */
2090static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2091{
2092 AssertPtr(pVM);
2093 AssertPtr(pVCpu);
2094 AssertPtr(pCpu);
2095 Assert(pCpu->idCpu != NIL_RTCPUID);
2096 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2097 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2098
2099 /*
2100 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2101 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2102 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2103 */
2104 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2105 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2106 {
2107 pVCpu->hm.s.fForceTLBFlush = true;
2108 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2109 }
2110
2111 /* Check for explicit TLB shootdown flushes. */
2112 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2113 {
2114 /*
2115 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2116 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2117 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2118 */
2119 pVCpu->hm.s.fForceTLBFlush = true;
2120 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2121 }
2122
2123 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2124 if (pVCpu->hm.s.fForceTLBFlush)
2125 {
2126 ++pCpu->uCurrentAsid;
2127 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2128 {
2129 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2130 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2131 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2132 }
2133
2134 pVCpu->hm.s.fForceTLBFlush = false;
2135 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2136 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2137 if (pCpu->fFlushAsidBeforeUse)
2138 {
2139 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2140 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2141 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2142 {
2143 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2144 pCpu->fFlushAsidBeforeUse = false;
2145 }
2146 else
2147 {
2148 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2149 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2150 }
2151 }
2152 }
2153 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2154 * where it is commented out. Support individual entry flushing
2155 * someday. */
2156#if 0
2157 else
2158 {
2159 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2160 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2161 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2162 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2163
2164 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2165 {
2166 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2167 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2168 {
2169 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2170 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2171 }
2172 else
2173 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2174
2175 pVCpu->hm.s.TlbShootdown.cPages = 0;
2176 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2177 }
2178 else
2179 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2180 }
2181#endif
2182
2183 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2184 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2185 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2186 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2187 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2188 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2189 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2190
2191 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2192 AssertRC(rc);
2193}
2194
2195
2196/**
2197 * Flushes the guest TLB entry based on CPU capabilities.
2198 *
2199 * @param pVCpu Pointer to the VMCPU.
2200 * @param pCpu Pointer to the global HM CPU struct.
2201 */
2202DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2203{
2204#ifdef HMVMX_ALWAYS_FLUSH_TLB
2205 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2206#endif
2207 PVM pVM = pVCpu->CTX_SUFF(pVM);
2208 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2209 {
2210 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2211 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2212 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2213 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2214 default:
2215 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2216 break;
2217 }
2218
2219 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2220 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2221
2222 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2223}
2224
2225
2226/**
2227 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2228 * TLB entries from the host TLB before VM-entry.
2229 *
2230 * @returns VBox status code.
2231 * @param pVM Pointer to the VM.
2232 */
2233static int hmR0VmxSetupTaggedTlb(PVM pVM)
2234{
2235 /*
2236 * Determine optimal flush type for Nested Paging.
2237 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2238 * guest execution (see hmR3InitFinalizeR0()).
2239 */
2240 if (pVM->hm.s.fNestedPaging)
2241 {
2242 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2243 {
2244 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2245 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2246 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2247 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2248 else
2249 {
2250 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2251 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2252 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2253 }
2254
2255 /* Make sure the write-back cacheable memory type for EPT is supported. */
2256 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2257 {
2258 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2259 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2261 }
2262 }
2263 else
2264 {
2265 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2266 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2267 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2268 }
2269 }
2270
2271 /*
2272 * Determine optimal flush type for VPID.
2273 */
2274 if (pVM->hm.s.vmx.fVpid)
2275 {
2276 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2277 {
2278 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2279 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2280 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2281 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2282 else
2283 {
2284 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2285 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2286 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2287 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2288 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2289 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2290 pVM->hm.s.vmx.fVpid = false;
2291 }
2292 }
2293 else
2294 {
2295 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2296 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2297 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2298 pVM->hm.s.vmx.fVpid = false;
2299 }
2300 }
2301
2302 /*
2303 * Setup the handler for flushing tagged-TLBs.
2304 */
2305 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2306 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2307 else if (pVM->hm.s.fNestedPaging)
2308 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2309 else if (pVM->hm.s.vmx.fVpid)
2310 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2311 else
2312 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2313 return VINF_SUCCESS;
2314}
2315
2316
2317/**
2318 * Sets up pin-based VM-execution controls in the VMCS.
2319 *
2320 * @returns VBox status code.
2321 * @param pVM Pointer to the VM.
2322 * @param pVCpu Pointer to the VMCPU.
2323 */
2324static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2325{
2326 AssertPtr(pVM);
2327 AssertPtr(pVCpu);
2328
2329 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2330 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2331
2332 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2333 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2334
2335 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2336 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2337
2338 /* Enable the VMX preemption timer. */
2339 if (pVM->hm.s.vmx.fUsePreemptTimer)
2340 {
2341 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2342 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2343 }
2344
2345 if ((val & zap) != val)
2346 {
2347 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2348 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2349 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2350 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2351 }
2352
2353 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2354 AssertRCReturn(rc, rc);
2355
2356 pVCpu->hm.s.vmx.u32PinCtls = val;
2357 return rc;
2358}
2359
2360
2361/**
2362 * Sets up processor-based VM-execution controls in the VMCS.
2363 *
2364 * @returns VBox status code.
2365 * @param pVM Pointer to the VM.
2366 * @param pVMCPU Pointer to the VMCPU.
2367 */
2368static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2369{
2370 AssertPtr(pVM);
2371 AssertPtr(pVCpu);
2372
2373 int rc = VERR_INTERNAL_ERROR_5;
2374 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2375 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2376
2377 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2378 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2379 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2380 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2381 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2382 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2383 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2384
2385 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2386 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2387 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2388 {
2389 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2390 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2391 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2392 }
2393
2394 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2395 if (!pVM->hm.s.fNestedPaging)
2396 {
2397 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2398 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2399 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2400 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2401 }
2402
2403 /* Use TPR shadowing if supported by the CPU. */
2404 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2405 {
2406 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2407 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2408 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2409 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2410 AssertRCReturn(rc, rc);
2411
2412 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2413 /* CR8 writes cause a VM-exit based on TPR threshold. */
2414 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2415 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2416 }
2417 else
2418 {
2419 /*
2420 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2421 * Set this control only for 64-bit guests.
2422 */
2423 if (pVM->hm.s.fAllow64BitGuests)
2424 {
2425 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2426 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2427 }
2428 }
2429
2430 /* Use MSR-bitmaps if supported by the CPU. */
2431 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2432 {
2433 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2434
2435 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2436 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2437 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2438 AssertRCReturn(rc, rc);
2439
2440 /*
2441 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2442 * automatically using dedicated fields in the VMCS.
2443 */
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2449
2450#if HC_ARCH_BITS == 64
2451 /*
2452 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2453 */
2454 if (pVM->hm.s.fAllow64BitGuests)
2455 {
2456 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2457 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2458 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2459 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2460 }
2461#endif
2462 }
2463
2464 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2465 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2466 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2467
2468 if ((val & zap) != val)
2469 {
2470 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2471 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2472 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2473 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2474 }
2475
2476 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2477 AssertRCReturn(rc, rc);
2478
2479 pVCpu->hm.s.vmx.u32ProcCtls = val;
2480
2481 /*
2482 * Secondary processor-based VM-execution controls.
2483 */
2484 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2485 {
2486 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2487 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2488
2489 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2490 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2491
2492 if (pVM->hm.s.fNestedPaging)
2493 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2494 else
2495 {
2496 /*
2497 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2498 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2499 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2500 */
2501 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2502 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2503 }
2504
2505 if (pVM->hm.s.vmx.fVpid)
2506 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2507
2508 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2509 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2510
2511 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2512 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2513 * done dynamically. */
2514 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2515 {
2516 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2517 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2518 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2519 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2520 AssertRCReturn(rc, rc);
2521 }
2522
2523 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2524 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2525
2526 if ((val & zap) != val)
2527 {
2528 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2529 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2530 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2531 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2532 }
2533
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2535 AssertRCReturn(rc, rc);
2536
2537 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2538 }
2539 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2540 {
2541 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2542 "available\n"));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 return VINF_SUCCESS;
2548}
2549
2550
2551/**
2552 * Sets up miscellaneous (everything other than Pin & Processor-based
2553 * VM-execution) control fields in the VMCS.
2554 *
2555 * @returns VBox status code.
2556 * @param pVM Pointer to the VM.
2557 * @param pVCpu Pointer to the VMCPU.
2558 */
2559static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2560{
2561 NOREF(pVM);
2562 AssertPtr(pVM);
2563 AssertPtr(pVCpu);
2564
2565 int rc = VERR_GENERAL_FAILURE;
2566
2567 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2568#if 0
2569 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2570 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2571 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2572
2573 /*
2574 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2575 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2576 * We thus use the exception bitmap to control it rather than use both.
2577 */
2578 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2580
2581 /** @todo Explore possibility of using IO-bitmaps. */
2582 /* All IO & IOIO instructions cause VM-exits. */
2583 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2584 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2585
2586 /* Initialize the MSR-bitmap area. */
2587 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2588 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2590#endif
2591
2592 /* Setup MSR auto-load/store area. */
2593 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2594 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2595 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2596 AssertRCReturn(rc, rc);
2597 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2598 AssertRCReturn(rc, rc);
2599
2600 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2601 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2602 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2603 AssertRCReturn(rc, rc);
2604
2605 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2606 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2607 AssertRCReturn(rc, rc);
2608
2609 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2610#if 0
2611 /* Setup debug controls */
2612 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2613 AssertRCReturn(rc, rc);
2614 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2615 AssertRCReturn(rc, rc);
2616#endif
2617
2618 return rc;
2619}
2620
2621
2622/**
2623 * Sets up the initial exception bitmap in the VMCS based on static conditions
2624 * (i.e. conditions that cannot ever change after starting the VM).
2625 *
2626 * @returns VBox status code.
2627 * @param pVM Pointer to the VM.
2628 * @param pVCpu Pointer to the VMCPU.
2629 */
2630static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2631{
2632 AssertPtr(pVM);
2633 AssertPtr(pVCpu);
2634
2635 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2636
2637 uint32_t u32XcptBitmap = 0;
2638
2639 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2640 if (!pVM->hm.s.fNestedPaging)
2641 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2642
2643 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2644 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2645 AssertRCReturn(rc, rc);
2646 return rc;
2647}
2648
2649
2650/**
2651 * Sets up the initial guest-state mask. The guest-state mask is consulted
2652 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2653 * for the nested virtualization case (as it would cause a VM-exit).
2654 *
2655 * @param pVCpu Pointer to the VMCPU.
2656 */
2657static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2658{
2659 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2660 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2661 return VINF_SUCCESS;
2662}
2663
2664
2665/**
2666 * Does per-VM VT-x initialization.
2667 *
2668 * @returns VBox status code.
2669 * @param pVM Pointer to the VM.
2670 */
2671VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2672{
2673 LogFlowFunc(("pVM=%p\n", pVM));
2674
2675 int rc = hmR0VmxStructsAlloc(pVM);
2676 if (RT_FAILURE(rc))
2677 {
2678 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2679 return rc;
2680 }
2681
2682 return VINF_SUCCESS;
2683}
2684
2685
2686/**
2687 * Does per-VM VT-x termination.
2688 *
2689 * @returns VBox status code.
2690 * @param pVM Pointer to the VM.
2691 */
2692VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2693{
2694 LogFlowFunc(("pVM=%p\n", pVM));
2695
2696#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2697 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2698 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2699#endif
2700 hmR0VmxStructsFree(pVM);
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Sets up the VM for execution under VT-x.
2707 * This function is only called once per-VM during initialization.
2708 *
2709 * @returns VBox status code.
2710 * @param pVM Pointer to the VM.
2711 */
2712VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2713{
2714 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2715 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2716
2717 LogFlowFunc(("pVM=%p\n", pVM));
2718
2719 /*
2720 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2721 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2722 */
2723 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2724 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2725 || !pVM->hm.s.vmx.pRealModeTSS))
2726 {
2727 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2728 return VERR_INTERNAL_ERROR;
2729 }
2730
2731#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2732 /*
2733 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2734 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2735 */
2736 if ( pVM->hm.s.fAllow64BitGuests
2737 && !HMVMX_IS_64BIT_HOST_MODE())
2738 {
2739 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2740 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2741 }
2742#endif
2743
2744 /* Initialize these always, see hmR3InitFinalizeR0().*/
2745 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2746 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2747
2748 /* Setup the tagged-TLB flush handlers. */
2749 int rc = hmR0VmxSetupTaggedTlb(pVM);
2750 if (RT_FAILURE(rc))
2751 {
2752 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2753 return rc;
2754 }
2755
2756 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2757 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2758#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2759 if ( HMVMX_IS_64BIT_HOST_MODE()
2760 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2761 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2762 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2763 {
2764 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2765 }
2766#endif
2767
2768 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2769 {
2770 PVMCPU pVCpu = &pVM->aCpus[i];
2771 AssertPtr(pVCpu);
2772 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2773
2774 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2775 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2776
2777 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2778 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2779 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2780
2781 /* Set revision dword at the beginning of the VMCS structure. */
2782 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2783
2784 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2785 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2787 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2788
2789 /* Load this VMCS as the current VMCS. */
2790 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2791 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2792 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2793
2794 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2796 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2797
2798 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2800 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2801
2802 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2804 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2805
2806 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2815 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2816 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2817 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2818#endif
2819
2820 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2821 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2822 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2823 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2824
2825 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2826
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2828 }
2829
2830 return VINF_SUCCESS;
2831}
2832
2833
2834/**
2835 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2836 * the VMCS.
2837 *
2838 * @returns VBox status code.
2839 * @param pVM Pointer to the VM.
2840 * @param pVCpu Pointer to the VMCPU.
2841 */
2842DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2843{
2844 NOREF(pVM); NOREF(pVCpu);
2845
2846 RTCCUINTREG uReg = ASMGetCR0();
2847 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2848 AssertRCReturn(rc, rc);
2849
2850#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2851 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2852 if (HMVMX_IS_64BIT_HOST_MODE())
2853 {
2854 uint64_t uRegCR3 = HMR0Get64bitCR3();
2855 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2856 }
2857 else
2858#endif
2859 {
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 }
2863 AssertRCReturn(rc, rc);
2864
2865 uReg = ASMGetCR4();
2866 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2867 AssertRCReturn(rc, rc);
2868 return rc;
2869}
2870
2871
2872#if HC_ARCH_BITS == 64
2873/**
2874 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2875 * requirements. See hmR0VmxSaveHostSegmentRegs().
2876 */
2877# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2878 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2879 { \
2880 bool fValidSelector = true; \
2881 if ((selValue) & X86_SEL_LDT) \
2882 { \
2883 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2884 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2885 } \
2886 if (fValidSelector) \
2887 { \
2888 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2889 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2890 } \
2891 (selValue) = 0; \
2892 }
2893#endif
2894
2895
2896/**
2897 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2898 * the host-state area in the VMCS.
2899 *
2900 * @returns VBox status code.
2901 * @param pVM Pointer to the VM.
2902 * @param pVCpu Pointer to the VMCPU.
2903 */
2904DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2905{
2906 int rc = VERR_INTERNAL_ERROR_5;
2907
2908#if HC_ARCH_BITS == 64
2909 /*
2910 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2911 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2912 */
2913 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2914 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2915#endif
2916
2917 /*
2918 * Host DS, ES, FS and GS segment registers.
2919 */
2920#if HC_ARCH_BITS == 64
2921 RTSEL uSelDS = ASMGetDS();
2922 RTSEL uSelES = ASMGetES();
2923 RTSEL uSelFS = ASMGetFS();
2924 RTSEL uSelGS = ASMGetGS();
2925#else
2926 RTSEL uSelDS = 0;
2927 RTSEL uSelES = 0;
2928 RTSEL uSelFS = 0;
2929 RTSEL uSelGS = 0;
2930#endif
2931
2932 /* Recalculate which host-state bits need to be manually restored. */
2933 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2934
2935 /*
2936 * Host CS and SS segment registers.
2937 */
2938#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2939 RTSEL uSelCS;
2940 RTSEL uSelSS;
2941 if (HMVMX_IS_64BIT_HOST_MODE())
2942 {
2943 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2944 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2945 }
2946 else
2947 {
2948 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2949 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2950 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2951 }
2952#else
2953 RTSEL uSelCS = ASMGetCS();
2954 RTSEL uSelSS = ASMGetSS();
2955#endif
2956
2957 /*
2958 * Host TR segment register.
2959 */
2960 RTSEL uSelTR = ASMGetTR();
2961
2962#if HC_ARCH_BITS == 64
2963 /*
2964 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2965 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2966 */
2967 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2968 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2969 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2970 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2971# undef VMXLOCAL_ADJUST_HOST_SEG
2972#endif
2973
2974 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2975 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2976 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2977 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2978 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2979 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2980 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2981 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2982 Assert(uSelCS);
2983 Assert(uSelTR);
2984
2985 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2986#if 0
2987 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2988 Assert(uSelSS != 0);
2989#endif
2990
2991 /* Write these host selector fields into the host-state area in the VMCS. */
2992 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2994#if HC_ARCH_BITS == 64
2995 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2999#endif
3000 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3001
3002 /*
3003 * Host GDTR and IDTR.
3004 */
3005 RTGDTR Gdtr;
3006 RT_ZERO(Gdtr);
3007#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3008 if (HMVMX_IS_64BIT_HOST_MODE())
3009 {
3010 X86XDTR64 Gdtr64;
3011 X86XDTR64 Idtr64;
3012 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3013 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3014 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3015
3016 Gdtr.cbGdt = Gdtr64.cb;
3017 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3018 }
3019 else
3020#endif
3021 {
3022 RTIDTR Idtr;
3023 ASMGetGDTR(&Gdtr);
3024 ASMGetIDTR(&Idtr);
3025 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3026 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3027
3028#if HC_ARCH_BITS == 64
3029 /*
3030 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3031 * maximum limit (0xffff) on every VM-exit.
3032 */
3033 if (Gdtr.cbGdt != 0xffff)
3034 {
3035 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3036 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3037 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3038 }
3039
3040 /*
3041 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3042 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3043 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3044 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3045 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3046 * hosts where we are pretty sure it won't cause trouble.
3047 */
3048# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3049 if (Idtr.cbIdt < 0x0fff)
3050# else
3051 if (Idtr.cbIdt != 0xffff)
3052# endif
3053 {
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3055 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3056 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3057 }
3058#endif
3059 }
3060
3061 /*
3062 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3063 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3064 */
3065 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3066 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3067 VERR_VMX_INVALID_HOST_STATE);
3068
3069 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3070#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3071 if (HMVMX_IS_64BIT_HOST_MODE())
3072 {
3073 /* We need the 64-bit TR base for hybrid darwin. */
3074 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3075 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3076 }
3077 else
3078#endif
3079 {
3080 uintptr_t uTRBase;
3081#if HC_ARCH_BITS == 64
3082 uTRBase = X86DESC64_BASE(pDesc);
3083
3084 /*
3085 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3086 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3087 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3088 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3089 *
3090 * [1] See Intel spec. 3.5 "System Descriptor Types".
3091 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3092 */
3093 Assert(pDesc->System.u4Type == 11);
3094 if ( pDesc->System.u16LimitLow != 0x67
3095 || pDesc->System.u4LimitHigh)
3096 {
3097 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3098 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3099 if (pVM->hm.s.uHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3100 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3101 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3102
3103 /* Store the GDTR here as we need it while restoring TR. */
3104 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3105 }
3106#else
3107 uTRBase = X86DESC_BASE(pDesc);
3108#endif
3109 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3110 }
3111 AssertRCReturn(rc, rc);
3112
3113 /*
3114 * Host FS base and GS base.
3115 */
3116#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3117 if (HMVMX_IS_64BIT_HOST_MODE())
3118 {
3119 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3120 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3121 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3122 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3123
3124# if HC_ARCH_BITS == 64
3125 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3126 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3127 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3128 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3129 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3130# endif
3131 }
3132#endif
3133 return rc;
3134}
3135
3136
3137/**
3138 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3139 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3140 * the host after every successful VM-exit.
3141 *
3142 * @returns VBox status code.
3143 * @param pVM Pointer to the VM.
3144 * @param pVCpu Pointer to the VMCPU.
3145 *
3146 * @remarks No-long-jump zone!!!
3147 */
3148DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3149{
3150 NOREF(pVM);
3151
3152 AssertPtr(pVCpu);
3153 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3154
3155 int rc = VINF_SUCCESS;
3156#if HC_ARCH_BITS == 64
3157 if (pVM->hm.s.fAllow64BitGuests)
3158 hmR0VmxLazySaveHostMsrs(pVCpu);
3159#endif
3160
3161 /*
3162 * Host Sysenter MSRs.
3163 */
3164 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3165 AssertRCReturn(rc, rc);
3166#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3167 if (HMVMX_IS_64BIT_HOST_MODE())
3168 {
3169 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3170 AssertRCReturn(rc, rc);
3171 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3172 }
3173 else
3174 {
3175 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3176 AssertRCReturn(rc, rc);
3177 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3178 }
3179#elif HC_ARCH_BITS == 32
3180 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3181 AssertRCReturn(rc, rc);
3182 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3183#else
3184 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3185 AssertRCReturn(rc, rc);
3186 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3187#endif
3188 AssertRCReturn(rc, rc);
3189
3190 /*
3191 * Host EFER MSR.
3192 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3193 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3194 */
3195 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3196 {
3197 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3198 AssertRCReturn(rc, rc);
3199 }
3200
3201 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3202 * hmR0VmxLoadGuestExitCtls() !! */
3203
3204 return rc;
3205}
3206
3207
3208/**
3209 * Figures out if we need to swap the EFER MSR which is
3210 * particularly expensive.
3211 *
3212 * We check all relevant bits. For now, that's everything
3213 * besides LMA/LME, as these two bits are handled by VM-entry,
3214 * see hmR0VmxLoadGuestExitCtls() and
3215 * hmR0VMxLoadGuestEntryCtls().
3216 *
3217 * @returns true if we need to load guest EFER, false otherwise.
3218 * @param pVCpu Pointer to the VMCPU.
3219 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3220 * out-of-sync. Make sure to update the required fields
3221 * before using them.
3222 *
3223 * @remarks Requires EFER, CR4.
3224 * @remarks No-long-jump zone!!!
3225 */
3226static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3227{
3228#ifdef HMVMX_ALWAYS_SWAP_EFER
3229 return true;
3230#endif
3231
3232#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3233 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3234 if (CPUMIsGuestInLongMode(pVCpu))
3235 return false;
3236#endif
3237
3238 PVM pVM = pVCpu->CTX_SUFF(pVM);
3239 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3240 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3241
3242 /*
3243 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3244 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3245 */
3246 if ( CPUMIsGuestInLongMode(pVCpu)
3247 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3248 {
3249 return true;
3250 }
3251
3252 /*
3253 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3254 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3255 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3256 */
3257 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3258 && (pMixedCtx->cr0 & X86_CR0_PG)
3259 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3260 {
3261 /* Assert that host is PAE capable. */
3262 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3263 return true;
3264 }
3265
3266 /** @todo Check the latest Intel spec. for any other bits,
3267 * like SMEP/SMAP? */
3268 return false;
3269}
3270
3271
3272/**
3273 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3274 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3275 * controls".
3276 *
3277 * @returns VBox status code.
3278 * @param pVCpu Pointer to the VMCPU.
3279 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3280 * out-of-sync. Make sure to update the required fields
3281 * before using them.
3282 *
3283 * @remarks Requires EFER.
3284 * @remarks No-long-jump zone!!!
3285 */
3286DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3287{
3288 int rc = VINF_SUCCESS;
3289 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3290 {
3291 PVM pVM = pVCpu->CTX_SUFF(pVM);
3292 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3293 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3294
3295 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3296 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3297
3298 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3299 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3300 {
3301 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3302 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3303 }
3304 else
3305 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3306
3307 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3308 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3309 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3310 {
3311 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3312 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3313 }
3314
3315 /*
3316 * The following should -not- be set (since we're not in SMM mode):
3317 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3318 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3319 */
3320
3321 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3322 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3323
3324 if ((val & zap) != val)
3325 {
3326 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3327 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3328 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3329 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3330 }
3331
3332 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3333 AssertRCReturn(rc, rc);
3334
3335 pVCpu->hm.s.vmx.u32EntryCtls = val;
3336 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3337 }
3338 return rc;
3339}
3340
3341
3342/**
3343 * Sets up the VM-exit controls in the VMCS.
3344 *
3345 * @returns VBox status code.
3346 * @param pVM Pointer to the VM.
3347 * @param pVCpu Pointer to the VMCPU.
3348 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3349 * out-of-sync. Make sure to update the required fields
3350 * before using them.
3351 *
3352 * @remarks Requires EFER.
3353 */
3354DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3355{
3356 NOREF(pMixedCtx);
3357
3358 int rc = VINF_SUCCESS;
3359 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3360 {
3361 PVM pVM = pVCpu->CTX_SUFF(pVM);
3362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3364
3365 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3366 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3367
3368 /*
3369 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3370 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3371 */
3372#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3373 if (HMVMX_IS_64BIT_HOST_MODE())
3374 {
3375 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3376 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3377 }
3378 else
3379 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3380#else
3381 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3382 {
3383 /* The switcher returns to long mode, EFER is managed by the switcher. */
3384 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3385 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3386 }
3387 else
3388 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3389#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3390
3391 /* If the newer VMCS fields for managing EFER exists, use it. */
3392 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3393 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3394 {
3395 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3396 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3397 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3398 }
3399
3400 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3401 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3402
3403 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3404 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3405 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3406
3407 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3408 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3409
3410 if ((val & zap) != val)
3411 {
3412 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3413 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3414 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3415 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3416 }
3417
3418 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3419 AssertRCReturn(rc, rc);
3420
3421 pVCpu->hm.s.vmx.u32ExitCtls = val;
3422 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3423 }
3424 return rc;
3425}
3426
3427
3428/**
3429 * Loads the guest APIC and related state.
3430 *
3431 * @returns VBox status code.
3432 * @param pVM Pointer to the VM.
3433 * @param pVCpu Pointer to the VMCPU.
3434 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3435 * out-of-sync. Make sure to update the required fields
3436 * before using them.
3437 */
3438DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3439{
3440 NOREF(pMixedCtx);
3441
3442 int rc = VINF_SUCCESS;
3443 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3444 {
3445 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3446 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3447 {
3448 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3449
3450 bool fPendingIntr = false;
3451 uint8_t u8Tpr = 0;
3452 uint8_t u8PendingIntr = 0;
3453 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3454 AssertRCReturn(rc, rc);
3455
3456 /*
3457 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3458 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3459 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3460 * the interrupt when we VM-exit for other reasons.
3461 */
3462 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3463 uint32_t u32TprThreshold = 0;
3464 if (fPendingIntr)
3465 {
3466 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3467 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3468 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3469 if (u8PendingPriority <= u8TprPriority)
3470 u32TprThreshold = u8PendingPriority;
3471 else
3472 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3473 }
3474 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3475
3476 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3477 AssertRCReturn(rc, rc);
3478 }
3479
3480 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3481 }
3482 return rc;
3483}
3484
3485
3486/**
3487 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3488 *
3489 * @returns Guest's interruptibility-state.
3490 * @param pVCpu Pointer to the VMCPU.
3491 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3492 * out-of-sync. Make sure to update the required fields
3493 * before using them.
3494 *
3495 * @remarks No-long-jump zone!!!
3496 */
3497DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3498{
3499 /*
3500 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3501 */
3502 uint32_t uIntrState = 0;
3503 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3504 {
3505 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3506 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3507 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3508 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3509 {
3510 if (pMixedCtx->eflags.Bits.u1IF)
3511 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3512 else
3513 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3514 }
3515 /* else: Although we can clear the force-flag here, let's keep this side-effects free. */
3516 }
3517
3518 /*
3519 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3520 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3521 * setting this would block host-NMIs and IRET will not clear the blocking.
3522 *
3523 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3524 */
3525 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3526 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3527 {
3528 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3529 }
3530
3531 return uIntrState;
3532}
3533
3534
3535/**
3536 * Loads the guest's interruptibility-state into the guest-state area in the
3537 * VMCS.
3538 *
3539 * @returns VBox status code.
3540 * @param pVCpu Pointer to the VMCPU.
3541 * @param uIntrState The interruptibility-state to set.
3542 */
3543static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3544{
3545 NOREF(pVCpu);
3546 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3547 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3548 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3549 AssertRCReturn(rc, rc);
3550 return rc;
3551}
3552
3553
3554/**
3555 * Loads the guest's RIP into the guest-state area in the VMCS.
3556 *
3557 * @returns VBox status code.
3558 * @param pVCpu Pointer to the VMCPU.
3559 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3560 * out-of-sync. Make sure to update the required fields
3561 * before using them.
3562 *
3563 * @remarks No-long-jump zone!!!
3564 */
3565static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3566{
3567 int rc = VINF_SUCCESS;
3568 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3569 {
3570 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3571 AssertRCReturn(rc, rc);
3572
3573 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3574 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3575 HMCPU_CF_VALUE(pVCpu)));
3576 }
3577 return rc;
3578}
3579
3580
3581/**
3582 * Loads the guest's RSP into the guest-state area in the VMCS.
3583 *
3584 * @returns VBox status code.
3585 * @param pVCpu Pointer to the VMCPU.
3586 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3587 * out-of-sync. Make sure to update the required fields
3588 * before using them.
3589 *
3590 * @remarks No-long-jump zone!!!
3591 */
3592static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3593{
3594 int rc = VINF_SUCCESS;
3595 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3596 {
3597 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3598 AssertRCReturn(rc, rc);
3599
3600 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3601 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3602 }
3603 return rc;
3604}
3605
3606
3607/**
3608 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3609 *
3610 * @returns VBox status code.
3611 * @param pVCpu Pointer to the VMCPU.
3612 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3613 * out-of-sync. Make sure to update the required fields
3614 * before using them.
3615 *
3616 * @remarks No-long-jump zone!!!
3617 */
3618static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3619{
3620 int rc = VINF_SUCCESS;
3621 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3622 {
3623 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3624 Let us assert it as such and use 32-bit VMWRITE. */
3625 Assert(!(pMixedCtx->rflags.u64 >> 32));
3626 X86EFLAGS Eflags = pMixedCtx->eflags;
3627 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3628 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3629 * These will never be cleared/set, unless some other part of the VMM
3630 * code is buggy - in which case we're better of finding and fixing
3631 * those bugs than hiding them. */
3632 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3633 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3634 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3635 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3636
3637 /*
3638 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3639 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3640 */
3641 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3642 {
3643 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3644 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3645 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3646 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3647 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3648 }
3649
3650 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3651 AssertRCReturn(rc, rc);
3652
3653 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3654 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3655 }
3656 return rc;
3657}
3658
3659
3660/**
3661 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3662 *
3663 * @returns VBox status code.
3664 * @param pVCpu Pointer to the VMCPU.
3665 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3666 * out-of-sync. Make sure to update the required fields
3667 * before using them.
3668 *
3669 * @remarks No-long-jump zone!!!
3670 */
3671DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3672{
3673 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3674 AssertRCReturn(rc, rc);
3675 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3676 AssertRCReturn(rc, rc);
3677 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3678 AssertRCReturn(rc, rc);
3679 return rc;
3680}
3681
3682
3683/**
3684 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3685 * CR0 is partially shared with the host and we have to consider the FPU bits.
3686 *
3687 * @returns VBox status code.
3688 * @param pVM Pointer to the VM.
3689 * @param pVCpu Pointer to the VMCPU.
3690 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3691 * out-of-sync. Make sure to update the required fields
3692 * before using them.
3693 *
3694 * @remarks No-long-jump zone!!!
3695 */
3696static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3697{
3698 /*
3699 * Guest CR0.
3700 * Guest FPU.
3701 */
3702 int rc = VINF_SUCCESS;
3703 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3704 {
3705 Assert(!(pMixedCtx->cr0 >> 32));
3706 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3707 PVM pVM = pVCpu->CTX_SUFF(pVM);
3708
3709 /* The guest's view (read access) of its CR0 is unblemished. */
3710 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3711 AssertRCReturn(rc, rc);
3712 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3713
3714 /* Setup VT-x's view of the guest CR0. */
3715 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3716 if (pVM->hm.s.fNestedPaging)
3717 {
3718 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3719 {
3720 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3721 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3723 }
3724 else
3725 {
3726 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3727 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3728 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729 }
3730
3731 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3732 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3733 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734
3735 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3736 AssertRCReturn(rc, rc);
3737 }
3738 else
3739 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3740
3741 /*
3742 * Guest FPU bits.
3743 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3744 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3745 */
3746 u32GuestCR0 |= X86_CR0_NE;
3747 bool fInterceptNM = false;
3748 if (CPUMIsGuestFPUStateActive(pVCpu))
3749 {
3750 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3751 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3752 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3753 }
3754 else
3755 {
3756 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3757 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3758 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3759 }
3760
3761 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3762 bool fInterceptMF = false;
3763 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3764 fInterceptMF = true;
3765
3766 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3767 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3768 {
3769 Assert(PDMVmmDevHeapIsEnabled(pVM));
3770 Assert(pVM->hm.s.vmx.pRealModeTSS);
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3772 fInterceptNM = true;
3773 fInterceptMF = true;
3774 }
3775 else
3776 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3777
3778 if (fInterceptNM)
3779 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3780 else
3781 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3782
3783 if (fInterceptMF)
3784 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3785 else
3786 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3787
3788 /* Additional intercepts for debugging, define these yourself explicitly. */
3789#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3790 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3791 | RT_BIT(X86_XCPT_BP)
3792 | RT_BIT(X86_XCPT_DB)
3793 | RT_BIT(X86_XCPT_DE)
3794 | RT_BIT(X86_XCPT_NM)
3795 | RT_BIT(X86_XCPT_TS)
3796 | RT_BIT(X86_XCPT_UD)
3797 | RT_BIT(X86_XCPT_NP)
3798 | RT_BIT(X86_XCPT_SS)
3799 | RT_BIT(X86_XCPT_GP)
3800 | RT_BIT(X86_XCPT_PF)
3801 | RT_BIT(X86_XCPT_MF)
3802 ;
3803#elif defined(HMVMX_ALWAYS_TRAP_PF)
3804 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3805#endif
3806
3807 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3808
3809 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3810 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3811 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3812 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3813 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3814 else
3815 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3816
3817 u32GuestCR0 |= uSetCR0;
3818 u32GuestCR0 &= uZapCR0;
3819 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3820
3821 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3822 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3823 AssertRCReturn(rc, rc);
3824 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3825 AssertRCReturn(rc, rc);
3826 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3827 uZapCR0));
3828
3829 /*
3830 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3831 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3832 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3833 */
3834 uint32_t u32CR0Mask = 0;
3835 u32CR0Mask = X86_CR0_PE
3836 | X86_CR0_NE
3837 | X86_CR0_WP
3838 | X86_CR0_PG
3839 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3840 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3841 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3842
3843 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3844 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3845 * and @bugref{6944}. */
3846#if 0
3847 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3848 u32CR0Mask &= ~X86_CR0_PE;
3849#endif
3850 if (pVM->hm.s.fNestedPaging)
3851 u32CR0Mask &= ~X86_CR0_WP;
3852
3853 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3854 if (fInterceptNM)
3855 {
3856 u32CR0Mask |= X86_CR0_TS
3857 | X86_CR0_MP;
3858 }
3859
3860 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3861 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3862 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3863 AssertRCReturn(rc, rc);
3864 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3865
3866 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3867 }
3868 return rc;
3869}
3870
3871
3872/**
3873 * Loads the guest control registers (CR3, CR4) into the guest-state area
3874 * in the VMCS.
3875 *
3876 * @returns VBox status code.
3877 * @param pVM Pointer to the VM.
3878 * @param pVCpu Pointer to the VMCPU.
3879 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3880 * out-of-sync. Make sure to update the required fields
3881 * before using them.
3882 *
3883 * @remarks No-long-jump zone!!!
3884 */
3885static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3886{
3887 int rc = VINF_SUCCESS;
3888 PVM pVM = pVCpu->CTX_SUFF(pVM);
3889
3890 /*
3891 * Guest CR2.
3892 * It's always loaded in the assembler code. Nothing to do here.
3893 */
3894
3895 /*
3896 * Guest CR3.
3897 */
3898 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3899 {
3900 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3901 if (pVM->hm.s.fNestedPaging)
3902 {
3903 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3904
3905 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3906 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3907 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3908 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3909
3910 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3911 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3912 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3913
3914 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3915 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3916 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3917 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3918
3919 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3920 AssertRCReturn(rc, rc);
3921 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3922
3923 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3924 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3925 {
3926 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3927 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3928 {
3929 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3930 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3931 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3932 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3933 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3934 }
3935
3936 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3937 have Unrestricted Execution to handle the guest when it's not using paging. */
3938 GCPhysGuestCR3 = pMixedCtx->cr3;
3939 }
3940 else
3941 {
3942 /*
3943 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3944 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3945 * EPT takes care of translating it to host-physical addresses.
3946 */
3947 RTGCPHYS GCPhys;
3948 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3949 Assert(PDMVmmDevHeapIsEnabled(pVM));
3950
3951 /* We obtain it here every time as the guest could have relocated this PCI region. */
3952 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3953 AssertRCReturn(rc, rc);
3954
3955 GCPhysGuestCR3 = GCPhys;
3956 }
3957
3958 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3959 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3960 }
3961 else
3962 {
3963 /* Non-nested paging case, just use the hypervisor's CR3. */
3964 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3965
3966 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3967 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3968 }
3969 AssertRCReturn(rc, rc);
3970
3971 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3972 }
3973
3974 /*
3975 * Guest CR4.
3976 */
3977 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3978 {
3979 Assert(!(pMixedCtx->cr4 >> 32));
3980 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3981
3982 /* The guest's view of its CR4 is unblemished. */
3983 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3984 AssertRCReturn(rc, rc);
3985 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3986
3987 /* Setup VT-x's view of the guest CR4. */
3988 /*
3989 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3990 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3991 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3992 */
3993 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3994 {
3995 Assert(pVM->hm.s.vmx.pRealModeTSS);
3996 Assert(PDMVmmDevHeapIsEnabled(pVM));
3997 u32GuestCR4 &= ~X86_CR4_VME;
3998 }
3999
4000 if (pVM->hm.s.fNestedPaging)
4001 {
4002 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4003 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4004 {
4005 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4006 u32GuestCR4 |= X86_CR4_PSE;
4007 /* Our identity mapping is a 32-bit page directory. */
4008 u32GuestCR4 &= ~X86_CR4_PAE;
4009 }
4010 /* else use guest CR4.*/
4011 }
4012 else
4013 {
4014 /*
4015 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4016 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4017 */
4018 switch (pVCpu->hm.s.enmShadowMode)
4019 {
4020 case PGMMODE_REAL: /* Real-mode. */
4021 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4022 case PGMMODE_32_BIT: /* 32-bit paging. */
4023 {
4024 u32GuestCR4 &= ~X86_CR4_PAE;
4025 break;
4026 }
4027
4028 case PGMMODE_PAE: /* PAE paging. */
4029 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4030 {
4031 u32GuestCR4 |= X86_CR4_PAE;
4032 break;
4033 }
4034
4035 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4036 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4037#ifdef VBOX_ENABLE_64_BITS_GUESTS
4038 break;
4039#endif
4040 default:
4041 AssertFailed();
4042 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4043 }
4044 }
4045
4046 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4047 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4048 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4049 u32GuestCR4 |= uSetCR4;
4050 u32GuestCR4 &= uZapCR4;
4051
4052 /* Write VT-x's view of the guest CR4 into the VMCS. */
4053 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4054 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4055 AssertRCReturn(rc, rc);
4056
4057 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4058 uint32_t u32CR4Mask = 0;
4059 u32CR4Mask = X86_CR4_VME
4060 | X86_CR4_PAE
4061 | X86_CR4_PGE
4062 | X86_CR4_PSE
4063 | X86_CR4_VMXE;
4064 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4065 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4066 AssertRCReturn(rc, rc);
4067
4068 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4069 }
4070 return rc;
4071}
4072
4073
4074/**
4075 * Loads the guest debug registers into the guest-state area in the VMCS.
4076 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4077 *
4078 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4079 *
4080 * @returns VBox status code.
4081 * @param pVCpu Pointer to the VMCPU.
4082 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4083 * out-of-sync. Make sure to update the required fields
4084 * before using them.
4085 *
4086 * @remarks No-long-jump zone!!!
4087 */
4088static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4089{
4090 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4091 return VINF_SUCCESS;
4092
4093#ifdef VBOX_STRICT
4094 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4095 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4096 {
4097 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4098 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4099 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4100 }
4101#endif
4102
4103 int rc;
4104 PVM pVM = pVCpu->CTX_SUFF(pVM);
4105 bool fInterceptDB = false;
4106 bool fInterceptMovDRx = false;
4107 if ( pVCpu->hm.s.fSingleInstruction
4108 || DBGFIsStepping(pVCpu))
4109 {
4110 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4111 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4112 {
4113 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4114 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4115 AssertRCReturn(rc, rc);
4116 Assert(fInterceptDB == false);
4117 }
4118 else
4119 {
4120 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4121 pVCpu->hm.s.fClearTrapFlag = true;
4122 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4123 fInterceptDB = true;
4124 }
4125 }
4126
4127 if ( fInterceptDB
4128 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4129 {
4130 /*
4131 * Use the combined guest and host DRx values found in the hypervisor
4132 * register set because the debugger has breakpoints active or someone
4133 * is single stepping on the host side without a monitor trap flag.
4134 *
4135 * Note! DBGF expects a clean DR6 state before executing guest code.
4136 */
4137#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4138 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4139 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4140 {
4141 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4142 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4143 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4144 }
4145 else
4146#endif
4147 if (!CPUMIsHyperDebugStateActive(pVCpu))
4148 {
4149 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4150 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4151 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4152 }
4153
4154 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4155 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4156 AssertRCReturn(rc, rc);
4157
4158 pVCpu->hm.s.fUsingHyperDR7 = true;
4159 fInterceptDB = true;
4160 fInterceptMovDRx = true;
4161 }
4162 else
4163 {
4164 /*
4165 * If the guest has enabled debug registers, we need to load them prior to
4166 * executing guest code so they'll trigger at the right time.
4167 */
4168 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4169 {
4170#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4171 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4172 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4173 {
4174 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4175 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4176 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4177 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4178 }
4179 else
4180#endif
4181 if (!CPUMIsGuestDebugStateActive(pVCpu))
4182 {
4183 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4184 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4185 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4186 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4187 }
4188 Assert(!fInterceptDB);
4189 Assert(!fInterceptMovDRx);
4190 }
4191 /*
4192 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4193 * must intercept #DB in order to maintain a correct DR6 guest value.
4194 */
4195#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4196 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4197 && !CPUMIsGuestDebugStateActive(pVCpu))
4198#else
4199 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4200#endif
4201 {
4202 fInterceptMovDRx = true;
4203 fInterceptDB = true;
4204 }
4205
4206 /* Update guest DR7. */
4207 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4208 AssertRCReturn(rc, rc);
4209
4210 pVCpu->hm.s.fUsingHyperDR7 = false;
4211 }
4212
4213 /*
4214 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4215 */
4216 if (fInterceptDB)
4217 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4218 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4219 {
4220#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4221 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4222#endif
4223 }
4224 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4225 AssertRCReturn(rc, rc);
4226
4227 /*
4228 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4229 */
4230 if (fInterceptMovDRx)
4231 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4232 else
4233 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4234 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4235 AssertRCReturn(rc, rc);
4236
4237 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4238 return VINF_SUCCESS;
4239}
4240
4241
4242#ifdef VBOX_STRICT
4243/**
4244 * Strict function to validate segment registers.
4245 *
4246 * @remarks ASSUMES CR0 is up to date.
4247 */
4248static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4249{
4250 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4251 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4252 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4253 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4254 && ( !CPUMIsGuestInRealModeEx(pCtx)
4255 && !CPUMIsGuestInV86ModeEx(pCtx)))
4256 {
4257 /* Protected mode checks */
4258 /* CS */
4259 Assert(pCtx->cs.Attr.n.u1Present);
4260 Assert(!(pCtx->cs.Attr.u & 0xf00));
4261 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4262 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4263 || !(pCtx->cs.Attr.n.u1Granularity));
4264 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4265 || (pCtx->cs.Attr.n.u1Granularity));
4266 /* CS cannot be loaded with NULL in protected mode. */
4267 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4268 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4269 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4270 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4271 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4272 else
4273 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4274 /* SS */
4275 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4276 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4277 if ( !(pCtx->cr0 & X86_CR0_PE)
4278 || pCtx->cs.Attr.n.u4Type == 3)
4279 {
4280 Assert(!pCtx->ss.Attr.n.u2Dpl);
4281 }
4282 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4283 {
4284 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4285 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4286 Assert(pCtx->ss.Attr.n.u1Present);
4287 Assert(!(pCtx->ss.Attr.u & 0xf00));
4288 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4289 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4290 || !(pCtx->ss.Attr.n.u1Granularity));
4291 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4292 || (pCtx->ss.Attr.n.u1Granularity));
4293 }
4294 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4295 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4296 {
4297 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4298 Assert(pCtx->ds.Attr.n.u1Present);
4299 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4300 Assert(!(pCtx->ds.Attr.u & 0xf00));
4301 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4302 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4303 || !(pCtx->ds.Attr.n.u1Granularity));
4304 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4305 || (pCtx->ds.Attr.n.u1Granularity));
4306 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4307 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4308 }
4309 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4310 {
4311 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4312 Assert(pCtx->es.Attr.n.u1Present);
4313 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4314 Assert(!(pCtx->es.Attr.u & 0xf00));
4315 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4316 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4317 || !(pCtx->es.Attr.n.u1Granularity));
4318 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4319 || (pCtx->es.Attr.n.u1Granularity));
4320 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4321 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4322 }
4323 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4324 {
4325 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4326 Assert(pCtx->fs.Attr.n.u1Present);
4327 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4328 Assert(!(pCtx->fs.Attr.u & 0xf00));
4329 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4330 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4331 || !(pCtx->fs.Attr.n.u1Granularity));
4332 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4333 || (pCtx->fs.Attr.n.u1Granularity));
4334 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4335 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4336 }
4337 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4338 {
4339 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4340 Assert(pCtx->gs.Attr.n.u1Present);
4341 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4342 Assert(!(pCtx->gs.Attr.u & 0xf00));
4343 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4344 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4345 || !(pCtx->gs.Attr.n.u1Granularity));
4346 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4347 || (pCtx->gs.Attr.n.u1Granularity));
4348 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4349 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4350 }
4351 /* 64-bit capable CPUs. */
4352# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4353 if (HMVMX_IS_64BIT_HOST_MODE())
4354 {
4355 Assert(!(pCtx->cs.u64Base >> 32));
4356 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4357 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4358 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4359 }
4360# endif
4361 }
4362 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4363 || ( CPUMIsGuestInRealModeEx(pCtx)
4364 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4365 {
4366 /* Real and v86 mode checks. */
4367 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4368 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4369 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4370 {
4371 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4372 }
4373 else
4374 {
4375 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4376 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4377 }
4378
4379 /* CS */
4380 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4381 Assert(pCtx->cs.u32Limit == 0xffff);
4382 Assert(u32CSAttr == 0xf3);
4383 /* SS */
4384 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4385 Assert(pCtx->ss.u32Limit == 0xffff);
4386 Assert(u32SSAttr == 0xf3);
4387 /* DS */
4388 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4389 Assert(pCtx->ds.u32Limit == 0xffff);
4390 Assert(u32DSAttr == 0xf3);
4391 /* ES */
4392 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4393 Assert(pCtx->es.u32Limit == 0xffff);
4394 Assert(u32ESAttr == 0xf3);
4395 /* FS */
4396 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4397 Assert(pCtx->fs.u32Limit == 0xffff);
4398 Assert(u32FSAttr == 0xf3);
4399 /* GS */
4400 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4401 Assert(pCtx->gs.u32Limit == 0xffff);
4402 Assert(u32GSAttr == 0xf3);
4403 /* 64-bit capable CPUs. */
4404# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4405 if (HMVMX_IS_64BIT_HOST_MODE())
4406 {
4407 Assert(!(pCtx->cs.u64Base >> 32));
4408 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4409 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4410 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4411 }
4412# endif
4413 }
4414}
4415#endif /* VBOX_STRICT */
4416
4417
4418/**
4419 * Writes a guest segment register into the guest-state area in the VMCS.
4420 *
4421 * @returns VBox status code.
4422 * @param pVCpu Pointer to the VMCPU.
4423 * @param idxSel Index of the selector in the VMCS.
4424 * @param idxLimit Index of the segment limit in the VMCS.
4425 * @param idxBase Index of the segment base in the VMCS.
4426 * @param idxAccess Index of the access rights of the segment in the VMCS.
4427 * @param pSelReg Pointer to the segment selector.
4428 *
4429 * @remarks No-long-jump zone!!!
4430 */
4431static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4432 uint32_t idxAccess, PCPUMSELREG pSelReg)
4433{
4434 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4435 AssertRCReturn(rc, rc);
4436 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4437 AssertRCReturn(rc, rc);
4438 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4439 AssertRCReturn(rc, rc);
4440
4441 uint32_t u32Access = pSelReg->Attr.u;
4442 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4443 {
4444 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4445 u32Access = 0xf3;
4446 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4447 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4448 }
4449 else
4450 {
4451 /*
4452 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4453 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4454 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4455 * loaded in protected-mode have their attribute as 0.
4456 */
4457 if (!u32Access)
4458 u32Access = X86DESCATTR_UNUSABLE;
4459 }
4460
4461 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4462 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4463 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4464
4465 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4466 AssertRCReturn(rc, rc);
4467 return rc;
4468}
4469
4470
4471/**
4472 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4473 * into the guest-state area in the VMCS.
4474 *
4475 * @returns VBox status code.
4476 * @param pVM Pointer to the VM.
4477 * @param pVCPU Pointer to the VMCPU.
4478 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4479 * out-of-sync. Make sure to update the required fields
4480 * before using them.
4481 *
4482 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4483 * @remarks No-long-jump zone!!!
4484 */
4485static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4486{
4487 int rc = VERR_INTERNAL_ERROR_5;
4488 PVM pVM = pVCpu->CTX_SUFF(pVM);
4489
4490 /*
4491 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4492 */
4493 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4494 {
4495 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4496 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4497 {
4498 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4501 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4502 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4503 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4504 }
4505
4506#ifdef VBOX_WITH_REM
4507 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4508 {
4509 Assert(pVM->hm.s.vmx.pRealModeTSS);
4510 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4511 if ( pVCpu->hm.s.vmx.fWasInRealMode
4512 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4513 {
4514 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4515 in real-mode (e.g. OpenBSD 4.0) */
4516 REMFlushTBs(pVM);
4517 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4518 pVCpu->hm.s.vmx.fWasInRealMode = false;
4519 }
4520 }
4521#endif
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4523 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4526 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4529 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4530 AssertRCReturn(rc, rc);
4531 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4532 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4533 AssertRCReturn(rc, rc);
4534 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4535 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4536 AssertRCReturn(rc, rc);
4537 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4538 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4539 AssertRCReturn(rc, rc);
4540
4541#ifdef VBOX_STRICT
4542 /* Validate. */
4543 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4544#endif
4545
4546 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4547 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4548 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4549 }
4550
4551 /*
4552 * Guest TR.
4553 */
4554 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4555 {
4556 /*
4557 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4558 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4559 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4560 */
4561 uint16_t u16Sel = 0;
4562 uint32_t u32Limit = 0;
4563 uint64_t u64Base = 0;
4564 uint32_t u32AccessRights = 0;
4565
4566 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4567 {
4568 u16Sel = pMixedCtx->tr.Sel;
4569 u32Limit = pMixedCtx->tr.u32Limit;
4570 u64Base = pMixedCtx->tr.u64Base;
4571 u32AccessRights = pMixedCtx->tr.Attr.u;
4572 }
4573 else
4574 {
4575 Assert(pVM->hm.s.vmx.pRealModeTSS);
4576 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4577
4578 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4579 RTGCPHYS GCPhys;
4580 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4581 AssertRCReturn(rc, rc);
4582
4583 X86DESCATTR DescAttr;
4584 DescAttr.u = 0;
4585 DescAttr.n.u1Present = 1;
4586 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4587
4588 u16Sel = 0;
4589 u32Limit = HM_VTX_TSS_SIZE;
4590 u64Base = GCPhys; /* in real-mode phys = virt. */
4591 u32AccessRights = DescAttr.u;
4592 }
4593
4594 /* Validate. */
4595 Assert(!(u16Sel & RT_BIT(2)));
4596 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4597 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4598 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4599 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4600 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4601 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4602 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4603 Assert( (u32Limit & 0xfff) == 0xfff
4604 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4605 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4606 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4607
4608 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4609 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4610 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4611 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4612
4613 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4614 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4615 }
4616
4617 /*
4618 * Guest GDTR.
4619 */
4620 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4621 {
4622 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4623 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4624
4625 /* Validate. */
4626 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4627
4628 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4629 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4630 }
4631
4632 /*
4633 * Guest LDTR.
4634 */
4635 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4636 {
4637 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4638 uint32_t u32Access = 0;
4639 if (!pMixedCtx->ldtr.Attr.u)
4640 u32Access = X86DESCATTR_UNUSABLE;
4641 else
4642 u32Access = pMixedCtx->ldtr.Attr.u;
4643
4644 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4645 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4646 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4647 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4648
4649 /* Validate. */
4650 if (!(u32Access & X86DESCATTR_UNUSABLE))
4651 {
4652 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4655 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4656 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4657 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4658 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4659 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4660 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4661 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4662 }
4663
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4665 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4666 }
4667
4668 /*
4669 * Guest IDTR.
4670 */
4671 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4672 {
4673 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4674 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4675
4676 /* Validate. */
4677 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4678
4679 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4680 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4681 }
4682
4683 return VINF_SUCCESS;
4684}
4685
4686
4687/**
4688 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4689 * areas. These MSRs will automatically be loaded to the host CPU on every
4690 * successful VM-entry and stored from the host CPU on every successful VM-exit.
4691 *
4692 * This also creates/updates MSR slots for the host MSRs. The actual host
4693 * MSR values are -not- updated here for performance reasons. See
4694 * hmR0VmxSaveHostMsrs().
4695 *
4696 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4697 *
4698 * @returns VBox status code.
4699 * @param pVCpu Pointer to the VMCPU.
4700 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4701 * out-of-sync. Make sure to update the required fields
4702 * before using them.
4703 *
4704 * @remarks No-long-jump zone!!!
4705 */
4706static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4707{
4708 AssertPtr(pVCpu);
4709 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4710
4711 /*
4712 * MSRs that we use the auto-load/store MSR area in the VMCS.
4713 */
4714 PVM pVM = pVCpu->CTX_SUFF(pVM);
4715 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4716 {
4717 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4718#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4719 if (pVM->hm.s.fAllow64BitGuests)
4720 {
4721 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4722 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4723 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4724 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4725# ifdef DEBUG
4726 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4727 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4728 {
4729 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4730 pMsr->u64Value));
4731 }
4732# endif
4733 }
4734#endif
4735 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4736 }
4737
4738 /*
4739 * Guest Sysenter MSRs.
4740 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4741 * VM-exits on WRMSRs for these MSRs.
4742 */
4743 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4744 {
4745 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4746 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4747 }
4748
4749 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4750 {
4751 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4752 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4753 }
4754
4755 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4756 {
4757 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4758 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4759 }
4760
4761 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4762 {
4763 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4764 {
4765 /*
4766 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4767 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4768 */
4769 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4770 {
4771 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4772 AssertRCReturn(rc,rc);
4773 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4774 }
4775 else
4776 {
4777 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4778 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4779 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4780 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4781 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4782 }
4783 }
4784 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4785 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4786 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4787 }
4788
4789 return VINF_SUCCESS;
4790}
4791
4792
4793/**
4794 * Loads the guest activity state into the guest-state area in the VMCS.
4795 *
4796 * @returns VBox status code.
4797 * @param pVCpu Pointer to the VMCPU.
4798 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4799 * out-of-sync. Make sure to update the required fields
4800 * before using them.
4801 *
4802 * @remarks No-long-jump zone!!!
4803 */
4804static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4805{
4806 NOREF(pCtx);
4807 /** @todo See if we can make use of other states, e.g.
4808 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4809 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4810 {
4811 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4812 AssertRCReturn(rc, rc);
4813
4814 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4815 }
4816 return VINF_SUCCESS;
4817}
4818
4819
4820/**
4821 * Sets up the appropriate function to run guest code.
4822 *
4823 * @returns VBox status code.
4824 * @param pVCpu Pointer to the VMCPU.
4825 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4826 * out-of-sync. Make sure to update the required fields
4827 * before using them.
4828 *
4829 * @remarks No-long-jump zone!!!
4830 */
4831static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4832{
4833 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4834 {
4835#ifndef VBOX_ENABLE_64_BITS_GUESTS
4836 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4837#endif
4838 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4839#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4840 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4841 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4842 {
4843 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4844 {
4845 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4846 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4847 | HM_CHANGED_VMX_EXIT_CTLS
4848 | HM_CHANGED_VMX_ENTRY_CTLS
4849 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4850 }
4851 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4852 }
4853#else
4854 /* 64-bit host or hybrid host. */
4855 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4856#endif
4857 }
4858 else
4859 {
4860 /* Guest is not in long mode, use the 32-bit handler. */
4861#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4862 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4863 {
4864 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4865 {
4866 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4867 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4868 | HM_CHANGED_VMX_EXIT_CTLS
4869 | HM_CHANGED_VMX_ENTRY_CTLS
4870 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4871 }
4872 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4873 }
4874#else
4875 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4876#endif
4877 }
4878 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4879 return VINF_SUCCESS;
4880}
4881
4882
4883/**
4884 * Wrapper for running the guest code in VT-x.
4885 *
4886 * @returns VBox strict status code.
4887 * @param pVM Pointer to the VM.
4888 * @param pVCpu Pointer to the VMCPU.
4889 * @param pCtx Pointer to the guest-CPU context.
4890 *
4891 * @remarks No-long-jump zone!!!
4892 */
4893DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4894{
4895 /*
4896 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4897 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4898 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4899 */
4900 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4901 /** @todo Add stats for resume vs launch. */
4902#ifdef VBOX_WITH_KERNEL_USING_XMM
4903 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4904#else
4905 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4906#endif
4907}
4908
4909
4910/**
4911 * Reports world-switch error and dumps some useful debug info.
4912 *
4913 * @param pVM Pointer to the VM.
4914 * @param pVCpu Pointer to the VMCPU.
4915 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4916 * @param pCtx Pointer to the guest-CPU context.
4917 * @param pVmxTransient Pointer to the VMX transient structure (only
4918 * exitReason updated).
4919 */
4920static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4921{
4922 Assert(pVM);
4923 Assert(pVCpu);
4924 Assert(pCtx);
4925 Assert(pVmxTransient);
4926 HMVMX_ASSERT_PREEMPT_SAFE();
4927
4928 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4929 switch (rcVMRun)
4930 {
4931 case VERR_VMX_INVALID_VMXON_PTR:
4932 AssertFailed();
4933 break;
4934 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4935 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4936 {
4937 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4938 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4939 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4940 AssertRC(rc);
4941
4942 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4943 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4944 Cannot do it here as we may have been long preempted. */
4945
4946#ifdef VBOX_STRICT
4947 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4948 pVmxTransient->uExitReason));
4949 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4950 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4951 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4952 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4953 else
4954 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4955 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4956 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4957
4958 /* VMX control bits. */
4959 uint32_t u32Val;
4960 uint64_t u64Val;
4961 HMVMXHCUINTREG uHCReg;
4962 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4963 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4964 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4965 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4970 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4971 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4974 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4975 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4976 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4977 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4978 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4979 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4980 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4981 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4982 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4983 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4984 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4985 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4986 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4987 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4992 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4993 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4994 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4995 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4996 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4997 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4998 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4999 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5000 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5001 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5002 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5003 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5004
5005 /* Guest bits. */
5006 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5007 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5008 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5009 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5010 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5011 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5012 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5013 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5014
5015 /* Host bits. */
5016 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5017 Log4(("Host CR0 %#RHr\n", uHCReg));
5018 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5019 Log4(("Host CR3 %#RHr\n", uHCReg));
5020 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5021 Log4(("Host CR4 %#RHr\n", uHCReg));
5022
5023 RTGDTR HostGdtr;
5024 PCX86DESCHC pDesc;
5025 ASMGetGDTR(&HostGdtr);
5026 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5027 Log4(("Host CS %#08x\n", u32Val));
5028 if (u32Val < HostGdtr.cbGdt)
5029 {
5030 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5031 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5032 }
5033
5034 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5035 Log4(("Host DS %#08x\n", u32Val));
5036 if (u32Val < HostGdtr.cbGdt)
5037 {
5038 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5039 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5040 }
5041
5042 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5043 Log4(("Host ES %#08x\n", u32Val));
5044 if (u32Val < HostGdtr.cbGdt)
5045 {
5046 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5047 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5048 }
5049
5050 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5051 Log4(("Host FS %#08x\n", u32Val));
5052 if (u32Val < HostGdtr.cbGdt)
5053 {
5054 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5055 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5056 }
5057
5058 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5059 Log4(("Host GS %#08x\n", u32Val));
5060 if (u32Val < HostGdtr.cbGdt)
5061 {
5062 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5063 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5064 }
5065
5066 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5067 Log4(("Host SS %#08x\n", u32Val));
5068 if (u32Val < HostGdtr.cbGdt)
5069 {
5070 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5071 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5072 }
5073
5074 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5075 Log4(("Host TR %#08x\n", u32Val));
5076 if (u32Val < HostGdtr.cbGdt)
5077 {
5078 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5079 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5080 }
5081
5082 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5083 Log4(("Host TR Base %#RHv\n", uHCReg));
5084 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5085 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5086 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5087 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5088 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5089 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5091 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5092 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5093 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5095 Log4(("Host RSP %#RHv\n", uHCReg));
5096 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5097 Log4(("Host RIP %#RHv\n", uHCReg));
5098# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5099 if (HMVMX_IS_64BIT_HOST_MODE())
5100 {
5101 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5102 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5103 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5104 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5105 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5106 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5107 }
5108# endif
5109#endif /* VBOX_STRICT */
5110 break;
5111 }
5112
5113 default:
5114 /* Impossible */
5115 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5116 break;
5117 }
5118 NOREF(pVM); NOREF(pCtx);
5119}
5120
5121
5122#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5123#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5124# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5125#endif
5126#ifdef VBOX_STRICT
5127static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5128{
5129 switch (idxField)
5130 {
5131 case VMX_VMCS_GUEST_RIP:
5132 case VMX_VMCS_GUEST_RSP:
5133 case VMX_VMCS_GUEST_SYSENTER_EIP:
5134 case VMX_VMCS_GUEST_SYSENTER_ESP:
5135 case VMX_VMCS_GUEST_GDTR_BASE:
5136 case VMX_VMCS_GUEST_IDTR_BASE:
5137 case VMX_VMCS_GUEST_CS_BASE:
5138 case VMX_VMCS_GUEST_DS_BASE:
5139 case VMX_VMCS_GUEST_ES_BASE:
5140 case VMX_VMCS_GUEST_FS_BASE:
5141 case VMX_VMCS_GUEST_GS_BASE:
5142 case VMX_VMCS_GUEST_SS_BASE:
5143 case VMX_VMCS_GUEST_LDTR_BASE:
5144 case VMX_VMCS_GUEST_TR_BASE:
5145 case VMX_VMCS_GUEST_CR3:
5146 return true;
5147 }
5148 return false;
5149}
5150
5151static bool hmR0VmxIsValidReadField(uint32_t idxField)
5152{
5153 switch (idxField)
5154 {
5155 /* Read-only fields. */
5156 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5157 return true;
5158 }
5159 /* Remaining readable fields should also be writable. */
5160 return hmR0VmxIsValidWriteField(idxField);
5161}
5162#endif /* VBOX_STRICT */
5163
5164
5165/**
5166 * Executes the specified handler in 64-bit mode.
5167 *
5168 * @returns VBox status code.
5169 * @param pVM Pointer to the VM.
5170 * @param pVCpu Pointer to the VMCPU.
5171 * @param pCtx Pointer to the guest CPU context.
5172 * @param enmOp The operation to perform.
5173 * @param cbParam Number of parameters.
5174 * @param paParam Array of 32-bit parameters.
5175 */
5176VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5177 uint32_t *paParam)
5178{
5179 int rc, rc2;
5180 PHMGLOBALCPUINFO pCpu;
5181 RTHCPHYS HCPhysCpuPage;
5182 RTCCUINTREG uOldEflags;
5183
5184 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5185 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5186 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5187 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5188
5189#ifdef VBOX_STRICT
5190 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5191 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5192
5193 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5194 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5195#endif
5196
5197 /* Disable interrupts. */
5198 uOldEflags = ASMIntDisableFlags();
5199
5200#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5201 RTCPUID idHostCpu = RTMpCpuId();
5202 CPUMR0SetLApic(pVCpu, idHostCpu);
5203#endif
5204
5205 pCpu = HMR0GetCurrentCpu();
5206 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5207
5208 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5209 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5210
5211 /* Leave VMX Root Mode. */
5212 VMXDisable();
5213
5214 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5215
5216 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5217 CPUMSetHyperEIP(pVCpu, enmOp);
5218 for (int i = (int)cbParam - 1; i >= 0; i--)
5219 CPUMPushHyper(pVCpu, paParam[i]);
5220
5221 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5222
5223 /* Call the switcher. */
5224 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5225 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5226
5227 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5228 /* Make sure the VMX instructions don't cause #UD faults. */
5229 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5230
5231 /* Re-enter VMX Root Mode */
5232 rc2 = VMXEnable(HCPhysCpuPage);
5233 if (RT_FAILURE(rc2))
5234 {
5235 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5236 ASMSetFlags(uOldEflags);
5237 return rc2;
5238 }
5239
5240 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5241 AssertRC(rc2);
5242 Assert(!(ASMGetFlags() & X86_EFL_IF));
5243 ASMSetFlags(uOldEflags);
5244 return rc;
5245}
5246
5247
5248/**
5249 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5250 * supporting 64-bit guests.
5251 *
5252 * @returns VBox status code.
5253 * @param fResume Whether to VMLAUNCH or VMRESUME.
5254 * @param pCtx Pointer to the guest-CPU context.
5255 * @param pCache Pointer to the VMCS cache.
5256 * @param pVM Pointer to the VM.
5257 * @param pVCpu Pointer to the VMCPU.
5258 */
5259DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5260{
5261 uint32_t aParam[6];
5262 PHMGLOBALCPUINFO pCpu = NULL;
5263 RTHCPHYS HCPhysCpuPage = 0;
5264 int rc = VERR_INTERNAL_ERROR_5;
5265
5266 pCpu = HMR0GetCurrentCpu();
5267 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5268
5269#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5270 pCache->uPos = 1;
5271 pCache->interPD = PGMGetInterPaeCR3(pVM);
5272 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5273#endif
5274
5275#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5276 pCache->TestIn.HCPhysCpuPage = 0;
5277 pCache->TestIn.HCPhysVmcs = 0;
5278 pCache->TestIn.pCache = 0;
5279 pCache->TestOut.HCPhysVmcs = 0;
5280 pCache->TestOut.pCache = 0;
5281 pCache->TestOut.pCtx = 0;
5282 pCache->TestOut.eflags = 0;
5283#endif
5284
5285 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5286 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5287 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5288 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5289 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5290 aParam[5] = 0;
5291
5292#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5293 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5294 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5295#endif
5296 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5297
5298#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5299 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5300 Assert(pCtx->dr[4] == 10);
5301 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5302#endif
5303
5304#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5305 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5306 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5307 pVCpu->hm.s.vmx.HCPhysVmcs));
5308 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5309 pCache->TestOut.HCPhysVmcs));
5310 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5311 pCache->TestOut.pCache));
5312 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5313 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5314 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5315 pCache->TestOut.pCtx));
5316 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5317#endif
5318 return rc;
5319}
5320
5321
5322/**
5323 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5324 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5325 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5326 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5327 *
5328 * @returns VBox status code.
5329 * @param pVM Pointer to the VM.
5330 * @param pVCpu Pointer to the VMCPU.
5331 */
5332static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5333{
5334#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5335{ \
5336 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5337 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5338 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5339 ++cReadFields; \
5340}
5341
5342 AssertPtr(pVM);
5343 AssertPtr(pVCpu);
5344 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5345 uint32_t cReadFields = 0;
5346
5347 /*
5348 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5349 * and serve to indicate exceptions to the rules.
5350 */
5351
5352 /* Guest-natural selector base fields. */
5353#if 0
5354 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5355 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5357#endif
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5368 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5370#if 0
5371 /* Unused natural width guest-state fields. */
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5374#endif
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5377
5378 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5379#if 0
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5389#endif
5390
5391 /* Natural width guest-state fields. */
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5393#if 0
5394 /* Currently unused field. */
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5396#endif
5397
5398 if (pVM->hm.s.fNestedPaging)
5399 {
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5401 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5402 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5403 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5404 }
5405 else
5406 {
5407 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5408 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5409 }
5410
5411#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5412 return VINF_SUCCESS;
5413}
5414
5415
5416/**
5417 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5418 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5419 * darwin, running 64-bit guests).
5420 *
5421 * @returns VBox status code.
5422 * @param pVCpu Pointer to the VMCPU.
5423 * @param idxField The VMCS field encoding.
5424 * @param u64Val 16, 32 or 64-bit value.
5425 */
5426VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5427{
5428 int rc;
5429 switch (idxField)
5430 {
5431 /*
5432 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5433 */
5434 /* 64-bit Control fields. */
5435 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5436 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5437 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5438 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5439 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5440 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5441 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5442 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5443 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5444 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5445 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5446 case VMX_VMCS64_CTRL_EPTP_FULL:
5447 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5448 /* 64-bit Guest-state fields. */
5449 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5450 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5451 case VMX_VMCS64_GUEST_PAT_FULL:
5452 case VMX_VMCS64_GUEST_EFER_FULL:
5453 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5454 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5455 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5456 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5457 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5458 /* 64-bit Host-state fields. */
5459 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5460 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5461 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5462 {
5463 rc = VMXWriteVmcs32(idxField, u64Val);
5464 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5465 break;
5466 }
5467
5468 /*
5469 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5470 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5471 */
5472 /* Natural-width Guest-state fields. */
5473 case VMX_VMCS_GUEST_CR3:
5474 case VMX_VMCS_GUEST_ES_BASE:
5475 case VMX_VMCS_GUEST_CS_BASE:
5476 case VMX_VMCS_GUEST_SS_BASE:
5477 case VMX_VMCS_GUEST_DS_BASE:
5478 case VMX_VMCS_GUEST_FS_BASE:
5479 case VMX_VMCS_GUEST_GS_BASE:
5480 case VMX_VMCS_GUEST_LDTR_BASE:
5481 case VMX_VMCS_GUEST_TR_BASE:
5482 case VMX_VMCS_GUEST_GDTR_BASE:
5483 case VMX_VMCS_GUEST_IDTR_BASE:
5484 case VMX_VMCS_GUEST_RSP:
5485 case VMX_VMCS_GUEST_RIP:
5486 case VMX_VMCS_GUEST_SYSENTER_ESP:
5487 case VMX_VMCS_GUEST_SYSENTER_EIP:
5488 {
5489 if (!(u64Val >> 32))
5490 {
5491 /* If this field is 64-bit, VT-x will zero out the top bits. */
5492 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5493 }
5494 else
5495 {
5496 /* Assert that only the 32->64 switcher case should ever come here. */
5497 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5498 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5499 }
5500 break;
5501 }
5502
5503 default:
5504 {
5505 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5506 rc = VERR_INVALID_PARAMETER;
5507 break;
5508 }
5509 }
5510 AssertRCReturn(rc, rc);
5511 return rc;
5512}
5513
5514
5515/**
5516 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5517 * hosts (except darwin) for 64-bit guests.
5518 *
5519 * @param pVCpu Pointer to the VMCPU.
5520 * @param idxField The VMCS field encoding.
5521 * @param u64Val 16, 32 or 64-bit value.
5522 */
5523VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5524{
5525 AssertPtr(pVCpu);
5526 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5527
5528 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5529 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5530
5531 /* Make sure there are no duplicates. */
5532 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5533 {
5534 if (pCache->Write.aField[i] == idxField)
5535 {
5536 pCache->Write.aFieldVal[i] = u64Val;
5537 return VINF_SUCCESS;
5538 }
5539 }
5540
5541 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5542 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5543 pCache->Write.cValidEntries++;
5544 return VINF_SUCCESS;
5545}
5546
5547/* Enable later when the assembly code uses these as callbacks. */
5548#if 0
5549/*
5550 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5551 *
5552 * @param pVCpu Pointer to the VMCPU.
5553 * @param pCache Pointer to the VMCS cache.
5554 *
5555 * @remarks No-long-jump zone!!!
5556 */
5557VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5558{
5559 AssertPtr(pCache);
5560 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5561 {
5562 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5563 AssertRC(rc);
5564 }
5565 pCache->Write.cValidEntries = 0;
5566}
5567
5568
5569/**
5570 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5571 *
5572 * @param pVCpu Pointer to the VMCPU.
5573 * @param pCache Pointer to the VMCS cache.
5574 *
5575 * @remarks No-long-jump zone!!!
5576 */
5577VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5578{
5579 AssertPtr(pCache);
5580 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5581 {
5582 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5583 AssertRC(rc);
5584 }
5585}
5586#endif
5587#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5588
5589
5590/**
5591 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5592 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5593 * timer.
5594 *
5595 * @returns VBox status code.
5596 * @param pVCpu Pointer to the VMCPU.
5597 *
5598 * @remarks No-long-jump zone!!!
5599 */
5600static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5601{
5602 int rc = VERR_INTERNAL_ERROR_5;
5603 bool fOffsettedTsc = false;
5604 bool fParavirtTsc = false;
5605 PVM pVM = pVCpu->CTX_SUFF(pVM);
5606 if (pVM->hm.s.vmx.fUsePreemptTimer)
5607 {
5608 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &fParavirtTsc,
5609 &pVCpu->hm.s.vmx.u64TSCOffset);
5610
5611 /* Make sure the returned values have sane upper and lower boundaries. */
5612 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5613 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5614 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5615 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5616
5617 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5618 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5619 }
5620 else
5621 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5622
5623 if (fParavirtTsc)
5624 {
5625 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5626 AssertRC(rc);
5627 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5628 }
5629
5630 if (fOffsettedTsc)
5631 {
5632 uint64_t u64CurTSC = ASMReadTSC();
5633 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5634 {
5635 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5636 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5637
5638 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5639 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5640 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5641 }
5642 else
5643 {
5644 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5645 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5646 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5647 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5648 }
5649 }
5650 else
5651 {
5652 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5653 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5654 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5655 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5656 }
5657}
5658
5659
5660/**
5661 * Determines if an exception is a contributory exception. Contributory
5662 * exceptions are ones which can cause double-faults. Page-fault is
5663 * intentionally not included here as it's a conditional contributory exception.
5664 *
5665 * @returns true if the exception is contributory, false otherwise.
5666 * @param uVector The exception vector.
5667 */
5668DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5669{
5670 switch (uVector)
5671 {
5672 case X86_XCPT_GP:
5673 case X86_XCPT_SS:
5674 case X86_XCPT_NP:
5675 case X86_XCPT_TS:
5676 case X86_XCPT_DE:
5677 return true;
5678 default:
5679 break;
5680 }
5681 return false;
5682}
5683
5684
5685/**
5686 * Sets an event as a pending event to be injected into the guest.
5687 *
5688 * @param pVCpu Pointer to the VMCPU.
5689 * @param u32IntInfo The VM-entry interruption-information field.
5690 * @param cbInstr The VM-entry instruction length in bytes (for software
5691 * interrupts, exceptions and privileged software
5692 * exceptions).
5693 * @param u32ErrCode The VM-entry exception error code.
5694 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5695 * page-fault.
5696 *
5697 * @remarks Statistics counter assumes this is a guest event being injected or
5698 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5699 * always incremented.
5700 */
5701DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5702 RTGCUINTPTR GCPtrFaultAddress)
5703{
5704 Assert(!pVCpu->hm.s.Event.fPending);
5705 pVCpu->hm.s.Event.fPending = true;
5706 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5707 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5708 pVCpu->hm.s.Event.cbInstr = cbInstr;
5709 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5710
5711 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5712}
5713
5714
5715/**
5716 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5717 *
5718 * @param pVCpu Pointer to the VMCPU.
5719 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5720 * out-of-sync. Make sure to update the required fields
5721 * before using them.
5722 */
5723DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5724{
5725 NOREF(pMixedCtx);
5726 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5727 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5728 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5729 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5730}
5731
5732
5733/**
5734 * Handle a condition that occurred while delivering an event through the guest
5735 * IDT.
5736 *
5737 * @returns VBox status code (informational error codes included).
5738 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5739 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5740 * continue execution of the guest which will delivery the #DF.
5741 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5742 *
5743 * @param pVCpu Pointer to the VMCPU.
5744 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5745 * out-of-sync. Make sure to update the required fields
5746 * before using them.
5747 * @param pVmxTransient Pointer to the VMX transient structure.
5748 *
5749 * @remarks No-long-jump zone!!!
5750 */
5751static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5752{
5753 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5754
5755 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5756 AssertRCReturn(rc, rc);
5757 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5758 {
5759 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5760 AssertRCReturn(rc, rc);
5761
5762 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5763 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5764
5765 typedef enum
5766 {
5767 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5768 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5769 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5770 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5771 } VMXREFLECTXCPT;
5772
5773 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5774 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5775 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5776 {
5777 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5778 {
5779 enmReflect = VMXREFLECTXCPT_XCPT;
5780#ifdef VBOX_STRICT
5781 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5782 && uExitVector == X86_XCPT_PF)
5783 {
5784 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5785 }
5786#endif
5787 if ( uExitVector == X86_XCPT_PF
5788 && uIdtVector == X86_XCPT_PF)
5789 {
5790 pVmxTransient->fVectoringDoublePF = true;
5791 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5792 }
5793 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5794 && hmR0VmxIsContributoryXcpt(uExitVector)
5795 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5796 || uIdtVector == X86_XCPT_PF))
5797 {
5798 enmReflect = VMXREFLECTXCPT_DF;
5799 }
5800 else if (uIdtVector == X86_XCPT_DF)
5801 enmReflect = VMXREFLECTXCPT_TF;
5802 }
5803 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5804 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5805 {
5806 /*
5807 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5808 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5809 */
5810 enmReflect = VMXREFLECTXCPT_XCPT;
5811
5812 if (uExitVector == X86_XCPT_PF)
5813 {
5814 pVmxTransient->fVectoringPF = true;
5815 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5816 }
5817 }
5818 }
5819 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5820 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5821 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5822 {
5823 /*
5824 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5825 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5826 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5827 */
5828 enmReflect = VMXREFLECTXCPT_XCPT;
5829 }
5830
5831 /*
5832 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5833 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5834 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5835 *
5836 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5837 */
5838 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5839 && enmReflect == VMXREFLECTXCPT_XCPT
5840 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5841 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5842 {
5843 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5844 }
5845
5846 switch (enmReflect)
5847 {
5848 case VMXREFLECTXCPT_XCPT:
5849 {
5850 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5851 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5852 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5853
5854 uint32_t u32ErrCode = 0;
5855 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5856 {
5857 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5858 AssertRCReturn(rc, rc);
5859 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5860 }
5861
5862 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5863 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5864 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5865 rc = VINF_SUCCESS;
5866 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5867 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5868
5869 break;
5870 }
5871
5872 case VMXREFLECTXCPT_DF:
5873 {
5874 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5875 rc = VINF_HM_DOUBLE_FAULT;
5876 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5877 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5878
5879 break;
5880 }
5881
5882 case VMXREFLECTXCPT_TF:
5883 {
5884 rc = VINF_EM_RESET;
5885 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5886 uExitVector));
5887 break;
5888 }
5889
5890 default:
5891 Assert(rc == VINF_SUCCESS);
5892 break;
5893 }
5894 }
5895 else if ( VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5896 && uExitVector != X86_XCPT_DF
5897 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5898 {
5899 /*
5900 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5901 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5902 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5903 */
5904 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5905 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5906 }
5907
5908 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5909 return rc;
5910}
5911
5912
5913/**
5914 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5915 *
5916 * @returns VBox status code.
5917 * @param pVCpu Pointer to the VMCPU.
5918 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5919 * out-of-sync. Make sure to update the required fields
5920 * before using them.
5921 *
5922 * @remarks No-long-jump zone!!!
5923 */
5924static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5925{
5926 NOREF(pMixedCtx);
5927
5928 /*
5929 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5930 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5931 */
5932 VMMRZCallRing3Disable(pVCpu);
5933 HM_DISABLE_PREEMPT_IF_NEEDED();
5934
5935 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5936 {
5937 uint32_t uVal = 0;
5938 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5939 AssertRCReturn(rc, rc);
5940
5941 uint32_t uShadow = 0;
5942 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5943 AssertRCReturn(rc, rc);
5944
5945 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5946 CPUMSetGuestCR0(pVCpu, uVal);
5947 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5948 }
5949
5950 HM_RESTORE_PREEMPT_IF_NEEDED();
5951 VMMRZCallRing3Enable(pVCpu);
5952 return VINF_SUCCESS;
5953}
5954
5955
5956/**
5957 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5958 *
5959 * @returns VBox status code.
5960 * @param pVCpu Pointer to the VMCPU.
5961 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5962 * out-of-sync. Make sure to update the required fields
5963 * before using them.
5964 *
5965 * @remarks No-long-jump zone!!!
5966 */
5967static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5968{
5969 NOREF(pMixedCtx);
5970
5971 int rc = VINF_SUCCESS;
5972 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5973 {
5974 uint32_t uVal = 0;
5975 uint32_t uShadow = 0;
5976 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5977 AssertRCReturn(rc, rc);
5978 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5979 AssertRCReturn(rc, rc);
5980
5981 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5982 CPUMSetGuestCR4(pVCpu, uVal);
5983 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5984 }
5985 return rc;
5986}
5987
5988
5989/**
5990 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5991 *
5992 * @returns VBox status code.
5993 * @param pVCpu Pointer to the VMCPU.
5994 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5995 * out-of-sync. Make sure to update the required fields
5996 * before using them.
5997 *
5998 * @remarks No-long-jump zone!!!
5999 */
6000static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6001{
6002 int rc = VINF_SUCCESS;
6003 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6004 {
6005 uint64_t u64Val = 0;
6006 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6007 AssertRCReturn(rc, rc);
6008
6009 pMixedCtx->rip = u64Val;
6010 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6011 }
6012 return rc;
6013}
6014
6015
6016/**
6017 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6018 *
6019 * @returns VBox status code.
6020 * @param pVCpu Pointer to the VMCPU.
6021 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6022 * out-of-sync. Make sure to update the required fields
6023 * before using them.
6024 *
6025 * @remarks No-long-jump zone!!!
6026 */
6027static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6028{
6029 int rc = VINF_SUCCESS;
6030 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6031 {
6032 uint64_t u64Val = 0;
6033 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6034 AssertRCReturn(rc, rc);
6035
6036 pMixedCtx->rsp = u64Val;
6037 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6038 }
6039 return rc;
6040}
6041
6042
6043/**
6044 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6045 *
6046 * @returns VBox status code.
6047 * @param pVCpu Pointer to the VMCPU.
6048 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6049 * out-of-sync. Make sure to update the required fields
6050 * before using them.
6051 *
6052 * @remarks No-long-jump zone!!!
6053 */
6054static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6055{
6056 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6057 {
6058 uint32_t uVal = 0;
6059 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6060 AssertRCReturn(rc, rc);
6061
6062 pMixedCtx->eflags.u32 = uVal;
6063 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6064 {
6065 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6066 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6067
6068 pMixedCtx->eflags.Bits.u1VM = 0;
6069 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6070 }
6071
6072 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6073 }
6074 return VINF_SUCCESS;
6075}
6076
6077
6078/**
6079 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6080 * guest-CPU context.
6081 */
6082DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6083{
6084 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6085 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6086 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6087 return rc;
6088}
6089
6090
6091/**
6092 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6093 * from the guest-state area in the VMCS.
6094 *
6095 * @param pVCpu Pointer to the VMCPU.
6096 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6097 * out-of-sync. Make sure to update the required fields
6098 * before using them.
6099 *
6100 * @remarks No-long-jump zone!!!
6101 */
6102static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6103{
6104 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6105 {
6106 uint32_t uIntrState = 0;
6107 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6108 AssertRC(rc);
6109
6110 if (!uIntrState)
6111 {
6112 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6113 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6114
6115 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6116 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6117 }
6118 else
6119 {
6120 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6121 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6122 {
6123 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6124 AssertRC(rc);
6125 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6126 AssertRC(rc);
6127
6128 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6129 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6130 }
6131 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6132 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6133
6134 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6135 {
6136 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6137 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6138 }
6139 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6140 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6141 }
6142
6143 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6144 }
6145}
6146
6147
6148/**
6149 * Saves the guest's activity state.
6150 *
6151 * @returns VBox status code.
6152 * @param pVCpu Pointer to the VMCPU.
6153 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6154 * out-of-sync. Make sure to update the required fields
6155 * before using them.
6156 *
6157 * @remarks No-long-jump zone!!!
6158 */
6159static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6160{
6161 NOREF(pMixedCtx);
6162 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6163 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6164 return VINF_SUCCESS;
6165}
6166
6167
6168/**
6169 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6170 * the current VMCS into the guest-CPU context.
6171 *
6172 * @returns VBox status code.
6173 * @param pVCpu Pointer to the VMCPU.
6174 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6175 * out-of-sync. Make sure to update the required fields
6176 * before using them.
6177 *
6178 * @remarks No-long-jump zone!!!
6179 */
6180static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6181{
6182 int rc = VINF_SUCCESS;
6183 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6184 {
6185 uint32_t u32Val = 0;
6186 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6187 pMixedCtx->SysEnter.cs = u32Val;
6188 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6189 }
6190
6191 uint64_t u64Val = 0;
6192 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6193 {
6194 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6195 pMixedCtx->SysEnter.eip = u64Val;
6196 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6197 }
6198 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6199 {
6200 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6201 pMixedCtx->SysEnter.esp = u64Val;
6202 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6203 }
6204 return rc;
6205}
6206
6207
6208/**
6209 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6210 * the CPU back into the guest-CPU context.
6211 *
6212 * @returns VBox status code.
6213 * @param pVCpu Pointer to the VMCPU.
6214 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6215 * out-of-sync. Make sure to update the required fields
6216 * before using them.
6217 *
6218 * @remarks No-long-jump zone!!!
6219 */
6220static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6221{
6222#if HC_ARCH_BITS == 64
6223 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6224 {
6225 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6226 VMMRZCallRing3Disable(pVCpu);
6227 HM_DISABLE_PREEMPT_IF_NEEDED();
6228
6229 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6230 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6231 {
6232 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6233 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6234 }
6235
6236 HM_RESTORE_PREEMPT_IF_NEEDED();
6237 VMMRZCallRing3Enable(pVCpu);
6238 }
6239 else
6240 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6241#else
6242 NOREF(pMixedCtx);
6243 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6244#endif
6245
6246 return VINF_SUCCESS;
6247}
6248
6249
6250/**
6251 * Saves the auto load/store'd guest MSRs from the current VMCS into
6252 * the guest-CPU context.
6253 *
6254 * @returns VBox status code.
6255 * @param pVCpu Pointer to the VMCPU.
6256 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6257 * out-of-sync. Make sure to update the required fields
6258 * before using them.
6259 *
6260 * @remarks No-long-jump zone!!!
6261 */
6262static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6263{
6264 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6265 return VINF_SUCCESS;
6266
6267 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6268 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6269 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6270 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6271 {
6272 switch (pMsr->u32Msr)
6273 {
6274 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6275 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6276 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6277 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6278 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6279 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6280 break;
6281
6282 default:
6283 {
6284 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6285 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6286 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6287 }
6288 }
6289 }
6290
6291 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6292 return VINF_SUCCESS;
6293}
6294
6295
6296/**
6297 * Saves the guest control registers from the current VMCS into the guest-CPU
6298 * context.
6299 *
6300 * @returns VBox status code.
6301 * @param pVCpu Pointer to the VMCPU.
6302 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6303 * out-of-sync. Make sure to update the required fields
6304 * before using them.
6305 *
6306 * @remarks No-long-jump zone!!!
6307 */
6308static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6309{
6310 /* Guest CR0. Guest FPU. */
6311 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6312 AssertRCReturn(rc, rc);
6313
6314 /* Guest CR4. */
6315 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6316 AssertRCReturn(rc, rc);
6317
6318 /* Guest CR2 - updated always during the world-switch or in #PF. */
6319 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6320 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6321 {
6322 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6323 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6324
6325 PVM pVM = pVCpu->CTX_SUFF(pVM);
6326 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6327 || ( pVM->hm.s.fNestedPaging
6328 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6329 {
6330 uint64_t u64Val = 0;
6331 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6332 if (pMixedCtx->cr3 != u64Val)
6333 {
6334 CPUMSetGuestCR3(pVCpu, u64Val);
6335 if (VMMRZCallRing3IsEnabled(pVCpu))
6336 {
6337 PGMUpdateCR3(pVCpu, u64Val);
6338 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6339 }
6340 else
6341 {
6342 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6343 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6344 }
6345 }
6346
6347 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6348 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6349 {
6350 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6351 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6352 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6353 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6354
6355 if (VMMRZCallRing3IsEnabled(pVCpu))
6356 {
6357 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6358 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6359 }
6360 else
6361 {
6362 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6363 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6364 }
6365 }
6366 }
6367
6368 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6369 }
6370
6371 /*
6372 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6373 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6374 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6375 *
6376 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6377 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6378 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6379 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6380 *
6381 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6382 */
6383 if (VMMRZCallRing3IsEnabled(pVCpu))
6384 {
6385 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6386 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6387
6388 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6389 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6390
6391 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6392 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6393 }
6394
6395 return rc;
6396}
6397
6398
6399/**
6400 * Reads a guest segment register from the current VMCS into the guest-CPU
6401 * context.
6402 *
6403 * @returns VBox status code.
6404 * @param pVCpu Pointer to the VMCPU.
6405 * @param idxSel Index of the selector in the VMCS.
6406 * @param idxLimit Index of the segment limit in the VMCS.
6407 * @param idxBase Index of the segment base in the VMCS.
6408 * @param idxAccess Index of the access rights of the segment in the VMCS.
6409 * @param pSelReg Pointer to the segment selector.
6410 *
6411 * @remarks No-long-jump zone!!!
6412 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6413 * macro as that takes care of whether to read from the VMCS cache or
6414 * not.
6415 */
6416DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6417 PCPUMSELREG pSelReg)
6418{
6419 NOREF(pVCpu);
6420
6421 uint32_t u32Val = 0;
6422 int rc = VMXReadVmcs32(idxSel, &u32Val);
6423 AssertRCReturn(rc, rc);
6424 pSelReg->Sel = (uint16_t)u32Val;
6425 pSelReg->ValidSel = (uint16_t)u32Val;
6426 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6427
6428 rc = VMXReadVmcs32(idxLimit, &u32Val);
6429 AssertRCReturn(rc, rc);
6430 pSelReg->u32Limit = u32Val;
6431
6432 uint64_t u64Val = 0;
6433 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6434 AssertRCReturn(rc, rc);
6435 pSelReg->u64Base = u64Val;
6436
6437 rc = VMXReadVmcs32(idxAccess, &u32Val);
6438 AssertRCReturn(rc, rc);
6439 pSelReg->Attr.u = u32Val;
6440
6441 /*
6442 * If VT-x marks the segment as unusable, most other bits remain undefined:
6443 * - For CS the L, D and G bits have meaning.
6444 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6445 * - For the remaining data segments no bits are defined.
6446 *
6447 * The present bit and the unusable bit has been observed to be set at the
6448 * same time (the selector was supposed to be invalid as we started executing
6449 * a V8086 interrupt in ring-0).
6450 *
6451 * What should be important for the rest of the VBox code, is that the P bit is
6452 * cleared. Some of the other VBox code recognizes the unusable bit, but
6453 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6454 * safe side here, we'll strip off P and other bits we don't care about. If
6455 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6456 *
6457 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6458 */
6459 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6460 {
6461 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6462
6463 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6464 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6465 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6466
6467 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6468#ifdef DEBUG_bird
6469 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6470 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6471 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6472#endif
6473 }
6474 return VINF_SUCCESS;
6475}
6476
6477
6478#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6479# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6480 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6481 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6482#else
6483# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6484 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6485 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6486#endif
6487
6488
6489/**
6490 * Saves the guest segment registers from the current VMCS into the guest-CPU
6491 * context.
6492 *
6493 * @returns VBox status code.
6494 * @param pVCpu Pointer to the VMCPU.
6495 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6496 * out-of-sync. Make sure to update the required fields
6497 * before using them.
6498 *
6499 * @remarks No-long-jump zone!!!
6500 */
6501static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6502{
6503 /* Guest segment registers. */
6504 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6505 {
6506 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6507 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6508 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6509 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6510 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6511 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6512 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6513
6514 /* Restore segment attributes for real-on-v86 mode hack. */
6515 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6516 {
6517 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6518 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6519 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6520 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6521 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6522 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6523 }
6524 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6525 }
6526
6527 return VINF_SUCCESS;
6528}
6529
6530
6531/**
6532 * Saves the guest descriptor table registers and task register from the current
6533 * VMCS into the guest-CPU context.
6534 *
6535 * @returns VBox status code.
6536 * @param pVCpu Pointer to the VMCPU.
6537 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6538 * out-of-sync. Make sure to update the required fields
6539 * before using them.
6540 *
6541 * @remarks No-long-jump zone!!!
6542 */
6543static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6544{
6545 int rc = VINF_SUCCESS;
6546
6547 /* Guest LDTR. */
6548 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6549 {
6550 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6551 AssertRCReturn(rc, rc);
6552 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6553 }
6554
6555 /* Guest GDTR. */
6556 uint64_t u64Val = 0;
6557 uint32_t u32Val = 0;
6558 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6559 {
6560 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6561 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6562 pMixedCtx->gdtr.pGdt = u64Val;
6563 pMixedCtx->gdtr.cbGdt = u32Val;
6564 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6565 }
6566
6567 /* Guest IDTR. */
6568 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6569 {
6570 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6571 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6572 pMixedCtx->idtr.pIdt = u64Val;
6573 pMixedCtx->idtr.cbIdt = u32Val;
6574 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6575 }
6576
6577 /* Guest TR. */
6578 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6579 {
6580 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6581 AssertRCReturn(rc, rc);
6582
6583 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6584 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6585 {
6586 rc = VMXLOCAL_READ_SEG(TR, tr);
6587 AssertRCReturn(rc, rc);
6588 }
6589 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6590 }
6591 return rc;
6592}
6593
6594#undef VMXLOCAL_READ_SEG
6595
6596
6597/**
6598 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6599 * context.
6600 *
6601 * @returns VBox status code.
6602 * @param pVCpu Pointer to the VMCPU.
6603 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6604 * out-of-sync. Make sure to update the required fields
6605 * before using them.
6606 *
6607 * @remarks No-long-jump zone!!!
6608 */
6609static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6610{
6611 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6612 {
6613 if (!pVCpu->hm.s.fUsingHyperDR7)
6614 {
6615 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6616 uint32_t u32Val;
6617 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6618 pMixedCtx->dr[7] = u32Val;
6619 }
6620
6621 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6622 }
6623 return VINF_SUCCESS;
6624}
6625
6626
6627/**
6628 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6629 *
6630 * @returns VBox status code.
6631 * @param pVCpu Pointer to the VMCPU.
6632 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6633 * out-of-sync. Make sure to update the required fields
6634 * before using them.
6635 *
6636 * @remarks No-long-jump zone!!!
6637 */
6638static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6639{
6640 NOREF(pMixedCtx);
6641
6642 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6643 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6644 return VINF_SUCCESS;
6645}
6646
6647
6648/**
6649 * Saves the entire guest state from the currently active VMCS into the
6650 * guest-CPU context. This essentially VMREADs all guest-data.
6651 *
6652 * @returns VBox status code.
6653 * @param pVCpu Pointer to the VMCPU.
6654 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6655 * out-of-sync. Make sure to update the required fields
6656 * before using them.
6657 */
6658static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6659{
6660 Assert(pVCpu);
6661 Assert(pMixedCtx);
6662
6663 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6664 return VINF_SUCCESS;
6665
6666 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6667 again on the ring-3 callback path, there is no real need to. */
6668 if (VMMRZCallRing3IsEnabled(pVCpu))
6669 VMMR0LogFlushDisable(pVCpu);
6670 else
6671 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6672 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6673
6674 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6675 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6676
6677 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6678 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6679
6680 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6681 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6682
6683 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6684 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6685
6686 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6687 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6688
6689 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6690 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6691
6692 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6693 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6694
6695 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6696 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6697
6698 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6699 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6700
6701 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6702 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6703
6704 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6705 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6706
6707 if (VMMRZCallRing3IsEnabled(pVCpu))
6708 VMMR0LogFlushEnable(pVCpu);
6709
6710 return rc;
6711}
6712
6713
6714/**
6715 * Check per-VM and per-VCPU force flag actions that require us to go back to
6716 * ring-3 for one reason or another.
6717 *
6718 * @returns VBox status code (information status code included).
6719 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6720 * ring-3.
6721 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6722 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6723 * interrupts)
6724 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6725 * all EMTs to be in ring-3.
6726 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6727 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6728 * to the EM loop.
6729 *
6730 * @param pVM Pointer to the VM.
6731 * @param pVCpu Pointer to the VMCPU.
6732 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6733 * out-of-sync. Make sure to update the required fields
6734 * before using them.
6735 */
6736static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6737{
6738 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6739
6740 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6741 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6742 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6743 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6744 {
6745 /* We need the control registers now, make sure the guest-CPU context is updated. */
6746 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6747 AssertRCReturn(rc3, rc3);
6748
6749 /* Pending HM CR3 sync. */
6750 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6751 {
6752 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6753 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6754 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6755 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6756 }
6757
6758 /* Pending HM PAE PDPEs. */
6759 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6760 {
6761 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6762 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6763 }
6764
6765 /* Pending PGM C3 sync. */
6766 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6767 {
6768 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6769 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6770 if (rc2 != VINF_SUCCESS)
6771 {
6772 AssertRC(rc2);
6773 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6774 return rc2;
6775 }
6776 }
6777
6778 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6779 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6780 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6781 {
6782 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6783 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6784 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6785 return rc2;
6786 }
6787
6788 /* Pending VM request packets, such as hardware interrupts. */
6789 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6790 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6791 {
6792 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6793 return VINF_EM_PENDING_REQUEST;
6794 }
6795
6796 /* Pending PGM pool flushes. */
6797 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6798 {
6799 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6800 return VINF_PGM_POOL_FLUSH_PENDING;
6801 }
6802
6803 /* Pending DMA requests. */
6804 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6805 {
6806 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6807 return VINF_EM_RAW_TO_R3;
6808 }
6809 }
6810
6811 return VINF_SUCCESS;
6812}
6813
6814
6815/**
6816 * Converts any TRPM trap into a pending HM event. This is typically used when
6817 * entering from ring-3 (not longjmp returns).
6818 *
6819 * @param pVCpu Pointer to the VMCPU.
6820 */
6821static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6822{
6823 Assert(TRPMHasTrap(pVCpu));
6824 Assert(!pVCpu->hm.s.Event.fPending);
6825
6826 uint8_t uVector;
6827 TRPMEVENT enmTrpmEvent;
6828 RTGCUINT uErrCode;
6829 RTGCUINTPTR GCPtrFaultAddress;
6830 uint8_t cbInstr;
6831
6832 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6833 AssertRC(rc);
6834
6835 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6836 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6837 if (enmTrpmEvent == TRPM_TRAP)
6838 {
6839 switch (uVector)
6840 {
6841 case X86_XCPT_NMI:
6842 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6843 break;
6844
6845 case X86_XCPT_BP:
6846 case X86_XCPT_OF:
6847 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6848 break;
6849
6850 case X86_XCPT_PF:
6851 case X86_XCPT_DF:
6852 case X86_XCPT_TS:
6853 case X86_XCPT_NP:
6854 case X86_XCPT_SS:
6855 case X86_XCPT_GP:
6856 case X86_XCPT_AC:
6857 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6858 /* no break! */
6859 default:
6860 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6861 break;
6862 }
6863 }
6864 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6865 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6866 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6867 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6868 else
6869 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6870
6871 rc = TRPMResetTrap(pVCpu);
6872 AssertRC(rc);
6873 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6874 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6875
6876 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6877 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6878}
6879
6880
6881/**
6882 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6883 * VT-x to execute any instruction.
6884 *
6885 * @param pvCpu Pointer to the VMCPU.
6886 */
6887static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6888{
6889 Assert(pVCpu->hm.s.Event.fPending);
6890
6891 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6892 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6893 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6894 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6895
6896 /* If a trap was already pending, we did something wrong! */
6897 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6898
6899 TRPMEVENT enmTrapType;
6900 switch (uVectorType)
6901 {
6902 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6903 enmTrapType = TRPM_HARDWARE_INT;
6904 break;
6905
6906 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6907 enmTrapType = TRPM_SOFTWARE_INT;
6908 break;
6909
6910 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6911 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6912 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6913 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6914 enmTrapType = TRPM_TRAP;
6915 break;
6916
6917 default:
6918 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6919 enmTrapType = TRPM_32BIT_HACK;
6920 break;
6921 }
6922
6923 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6924
6925 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6926 AssertRC(rc);
6927
6928 if (fErrorCodeValid)
6929 TRPMSetErrorCode(pVCpu, uErrorCode);
6930
6931 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6932 && uVector == X86_XCPT_PF)
6933 {
6934 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6935 }
6936 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6937 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6938 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6939 {
6940 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6941 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6942 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6943 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6944 }
6945 pVCpu->hm.s.Event.fPending = false;
6946}
6947
6948
6949/**
6950 * Does the necessary state syncing before returning to ring-3 for any reason
6951 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6952 *
6953 * @returns VBox status code.
6954 * @param pVM Pointer to the VM.
6955 * @param pVCpu Pointer to the VMCPU.
6956 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6957 * be out-of-sync. Make sure to update the required
6958 * fields before using them.
6959 * @param fSaveGuestState Whether to save the guest state or not.
6960 *
6961 * @remarks No-long-jmp zone!!!
6962 */
6963static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6964{
6965 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6966 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6967
6968 RTCPUID idCpu = RTMpCpuId();
6969 Log4Func(("HostCpuId=%u\n", idCpu));
6970
6971 /*
6972 * !!! IMPORTANT !!!
6973 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6974 */
6975
6976 /* Save the guest state if necessary. */
6977 if ( fSaveGuestState
6978 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6979 {
6980 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6981 AssertRCReturn(rc, rc);
6982 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6983 }
6984
6985 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6986 if (CPUMIsGuestFPUStateActive(pVCpu))
6987 {
6988 /* We shouldn't reload CR0 without saving it first. */
6989 if (!fSaveGuestState)
6990 {
6991 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6992 AssertRCReturn(rc, rc);
6993 }
6994 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6995 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6996 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6997 }
6998
6999 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7000#ifdef VBOX_STRICT
7001 if (CPUMIsHyperDebugStateActive(pVCpu))
7002 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7003#endif
7004 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7005 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7006 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7007 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7008
7009#if HC_ARCH_BITS == 64
7010 /* Restore host-state bits that VT-x only restores partially. */
7011 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7012 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7013 {
7014 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7015 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7016 }
7017 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7018#endif
7019
7020#if HC_ARCH_BITS == 64
7021 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7022 if ( pVM->hm.s.fAllow64BitGuests
7023 && pVCpu->hm.s.vmx.fLazyMsrs)
7024 {
7025 /* We shouldn't reload the guest MSRs without saving it first. */
7026 if (!fSaveGuestState)
7027 {
7028 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7029 AssertRCReturn(rc, rc);
7030 }
7031 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7032 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7033 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7034 }
7035#endif
7036
7037 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7038 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7039
7040 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7041 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7042 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7043 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7044 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7045 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7046 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7047 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7048
7049 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7050
7051 /** @todo This partially defeats the purpose of having preemption hooks.
7052 * The problem is, deregistering the hooks should be moved to a place that
7053 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7054 * context.
7055 */
7056 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7057 {
7058 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7059 AssertRCReturn(rc, rc);
7060
7061 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7062 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7063 }
7064 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7065 NOREF(idCpu);
7066
7067 return VINF_SUCCESS;
7068}
7069
7070
7071/**
7072 * Leaves the VT-x session.
7073 *
7074 * @returns VBox status code.
7075 * @param pVM Pointer to the VM.
7076 * @param pVCpu Pointer to the VMCPU.
7077 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7078 * out-of-sync. Make sure to update the required fields
7079 * before using them.
7080 *
7081 * @remarks No-long-jmp zone!!!
7082 */
7083DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7084{
7085 HM_DISABLE_PREEMPT_IF_NEEDED();
7086 HMVMX_ASSERT_CPU_SAFE();
7087 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7088 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7089
7090 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7091 and done this from the VMXR0ThreadCtxCallback(). */
7092 if (!pVCpu->hm.s.fLeaveDone)
7093 {
7094 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7095 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7096 pVCpu->hm.s.fLeaveDone = true;
7097 }
7098 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7099
7100 /*
7101 * !!! IMPORTANT !!!
7102 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7103 */
7104
7105 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7106 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7107 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7108 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7109 VMMR0ThreadCtxHooksDeregister(pVCpu);
7110
7111 /* Leave HM context. This takes care of local init (term). */
7112 int rc = HMR0LeaveCpu(pVCpu);
7113
7114 HM_RESTORE_PREEMPT_IF_NEEDED();
7115
7116 return rc;
7117}
7118
7119
7120/**
7121 * Does the necessary state syncing before doing a longjmp to ring-3.
7122 *
7123 * @returns VBox status code.
7124 * @param pVM Pointer to the VM.
7125 * @param pVCpu Pointer to the VMCPU.
7126 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7127 * out-of-sync. Make sure to update the required fields
7128 * before using them.
7129 *
7130 * @remarks No-long-jmp zone!!!
7131 */
7132DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7133{
7134 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7135}
7136
7137
7138/**
7139 * Take necessary actions before going back to ring-3.
7140 *
7141 * An action requires us to go back to ring-3. This function does the necessary
7142 * steps before we can safely return to ring-3. This is not the same as longjmps
7143 * to ring-3, this is voluntary and prepares the guest so it may continue
7144 * executing outside HM (recompiler/IEM).
7145 *
7146 * @returns VBox status code.
7147 * @param pVM Pointer to the VM.
7148 * @param pVCpu Pointer to the VMCPU.
7149 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7150 * out-of-sync. Make sure to update the required fields
7151 * before using them.
7152 * @param rcExit The reason for exiting to ring-3. Can be
7153 * VINF_VMM_UNKNOWN_RING3_CALL.
7154 */
7155static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7156{
7157 Assert(pVM);
7158 Assert(pVCpu);
7159 Assert(pMixedCtx);
7160 HMVMX_ASSERT_PREEMPT_SAFE();
7161
7162 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7163 {
7164 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7165 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7166 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7167 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7168 }
7169
7170 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7171 VMMRZCallRing3Disable(pVCpu);
7172 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7173
7174 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7175 if (pVCpu->hm.s.Event.fPending)
7176 {
7177 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7178 Assert(!pVCpu->hm.s.Event.fPending);
7179 }
7180
7181 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7182 and if we're injecting an event we should have a TRPM trap pending. */
7183 Assert(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu));
7184 Assert(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu));
7185
7186 /* Save guest state and restore host state bits. */
7187 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7188 AssertRCReturn(rc, rc);
7189 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7190 /* Thread-context hooks are unregistered at this point!!! */
7191
7192 /* Sync recompiler state. */
7193 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7194 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7195 | CPUM_CHANGED_LDTR
7196 | CPUM_CHANGED_GDTR
7197 | CPUM_CHANGED_IDTR
7198 | CPUM_CHANGED_TR
7199 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7200 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7201 if ( pVM->hm.s.fNestedPaging
7202 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7203 {
7204 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7205 }
7206
7207 Assert(!pVCpu->hm.s.fClearTrapFlag);
7208
7209 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7210 if (rcExit != VINF_EM_RAW_INTERRUPT)
7211 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7212
7213 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7214
7215 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7216 VMMRZCallRing3RemoveNotification(pVCpu);
7217 VMMRZCallRing3Enable(pVCpu);
7218
7219 return rc;
7220}
7221
7222
7223/**
7224 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7225 * longjump to ring-3 and possibly get preempted.
7226 *
7227 * @returns VBox status code.
7228 * @param pVCpu Pointer to the VMCPU.
7229 * @param enmOperation The operation causing the ring-3 longjump.
7230 * @param pvUser Opaque pointer to the guest-CPU context. The data
7231 * may be out-of-sync. Make sure to update the required
7232 * fields before using them.
7233 */
7234DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7235{
7236 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7237 {
7238 /*
7239 * !!! IMPORTANT !!!
7240 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7241 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7242 */
7243 VMMRZCallRing3RemoveNotification(pVCpu);
7244 VMMRZCallRing3Disable(pVCpu);
7245 HM_DISABLE_PREEMPT_IF_NEEDED();
7246
7247 PVM pVM = pVCpu->CTX_SUFF(pVM);
7248 if (CPUMIsGuestFPUStateActive(pVCpu))
7249 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7250
7251 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7252
7253#if HC_ARCH_BITS == 64
7254 /* Restore host-state bits that VT-x only restores partially. */
7255 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7256 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7257 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7258 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7259
7260 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7261 if ( pVM->hm.s.fAllow64BitGuests
7262 && pVCpu->hm.s.vmx.fLazyMsrs)
7263 {
7264 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7265 }
7266#endif
7267 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7268 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7269 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7270 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7271 {
7272 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7273 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7274 }
7275
7276 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7277 VMMR0ThreadCtxHooksDeregister(pVCpu);
7278
7279 HMR0LeaveCpu(pVCpu);
7280 HM_RESTORE_PREEMPT_IF_NEEDED();
7281 return VINF_SUCCESS;
7282 }
7283
7284 Assert(pVCpu);
7285 Assert(pvUser);
7286 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7287 HMVMX_ASSERT_PREEMPT_SAFE();
7288
7289 VMMRZCallRing3Disable(pVCpu);
7290 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7291
7292 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7293 enmOperation));
7294
7295 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7296 AssertRCReturn(rc, rc);
7297
7298 VMMRZCallRing3Enable(pVCpu);
7299 return VINF_SUCCESS;
7300}
7301
7302
7303/**
7304 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7305 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7306 *
7307 * @param pVCpu Pointer to the VMCPU.
7308 */
7309DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7310{
7311 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7312 {
7313 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7314 {
7315 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7316 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7317 AssertRC(rc);
7318 Log4(("Setup interrupt-window exiting\n"));
7319 }
7320 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7321}
7322
7323
7324/**
7325 * Clears the interrupt-window exiting control in the VMCS.
7326 *
7327 * @param pVCpu Pointer to the VMCPU.
7328 */
7329DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7330{
7331 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7332 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7333 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7334 AssertRC(rc);
7335 Log4(("Cleared interrupt-window exiting\n"));
7336}
7337
7338
7339/**
7340 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7341 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7342 *
7343 * @param pVCpu Pointer to the VMCPU.
7344 */
7345DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7346{
7347 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7348 {
7349 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7350 {
7351 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7352 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7353 AssertRC(rc);
7354 Log4(("Setup NMI-window exiting\n"));
7355 }
7356 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7357}
7358
7359
7360/**
7361 * Clears the NMI-window exiting control in the VMCS.
7362 *
7363 * @param pVCpu Pointer to the VMCPU.
7364 */
7365DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7366{
7367 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7368 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7369 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7370 AssertRC(rc);
7371 Log4(("Cleared NMI-window exiting\n"));
7372}
7373
7374
7375/**
7376 * Evaluates the event to be delivered to the guest and sets it as the pending
7377 * event.
7378 *
7379 * @param pVCpu Pointer to the VMCPU.
7380 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7381 * out-of-sync. Make sure to update the required fields
7382 * before using them.
7383 */
7384static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7385{
7386 Assert(!pVCpu->hm.s.Event.fPending);
7387
7388 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7389 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7390 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7391 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7392 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7393
7394 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7395 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7396 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7397 Assert(!TRPMHasTrap(pVCpu));
7398
7399 /*
7400 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7401 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7402 */
7403 /** @todo SMI. SMIs take priority over NMIs. */
7404 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7405 {
7406 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7407 if ( !fBlockNmi
7408 && !fBlockSti
7409 && !fBlockMovSS)
7410 {
7411 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7412 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7413 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7414
7415 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7416 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7417 }
7418 else
7419 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7420 }
7421 /*
7422 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7423 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7424 */
7425 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7426 && !pVCpu->hm.s.fSingleInstruction)
7427 {
7428 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7429 AssertRC(rc);
7430 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7431 if ( !fBlockInt
7432 && !fBlockSti
7433 && !fBlockMovSS)
7434 {
7435 uint8_t u8Interrupt;
7436 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7437 if (RT_SUCCESS(rc))
7438 {
7439 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7440 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7441 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7442
7443 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7444 }
7445 else
7446 {
7447 /** @todo Does this actually happen? If not turn it into an assertion. */
7448 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7449 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7450 }
7451 }
7452 else
7453 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7454 }
7455}
7456
7457
7458/**
7459 * Sets a pending-debug exception to be delivered to the guest if the guest is
7460 * single-stepping.
7461 *
7462 * @param pVCpu Pointer to the VMCPU.
7463 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7464 * out-of-sync. Make sure to update the required fields
7465 * before using them.
7466 */
7467DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7468{
7469 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7470 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7471 {
7472 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7473 AssertRC(rc);
7474 }
7475}
7476
7477
7478/**
7479 * Injects any pending events into the guest if the guest is in a state to
7480 * receive them.
7481 *
7482 * @returns VBox status code (informational status codes included).
7483 * @param pVCpu Pointer to the VMCPU.
7484 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7485 * out-of-sync. Make sure to update the required fields
7486 * before using them.
7487 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7488 * return VINF_EM_DBG_STEPPED if the event was
7489 * dispatched directly.
7490 */
7491static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7492{
7493 HMVMX_ASSERT_PREEMPT_SAFE();
7494 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7495
7496 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7497 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7498 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7499 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7500
7501 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7502 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7503 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7504 Assert(!TRPMHasTrap(pVCpu));
7505
7506 int rc = VINF_SUCCESS;
7507 if (pVCpu->hm.s.Event.fPending)
7508 {
7509 /*
7510 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7511 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7512 * ended up enabling interrupts outside VT-x.
7513 */
7514 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7515 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7516 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7517 {
7518 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7519 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7520 }
7521
7522#ifdef VBOX_STRICT
7523 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7524 {
7525 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7526 Assert(!fBlockInt);
7527 Assert(!fBlockSti);
7528 Assert(!fBlockMovSS);
7529 }
7530 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7531 {
7532 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7533 Assert(!fBlockSti);
7534 Assert(!fBlockMovSS);
7535 Assert(!fBlockNmi);
7536 }
7537#endif
7538 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7539 (uint8_t)uIntType));
7540 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7541 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping, &uIntrState);
7542 AssertRCReturn(rc, rc);
7543
7544 /* Update the interruptibility-state as it could have been changed by
7545 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7546 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7547 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7548
7549#ifdef VBOX_WITH_STATISTICS
7550 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7551 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7552 else
7553 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7554#endif
7555 }
7556
7557 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7558 if ( fBlockSti
7559 || fBlockMovSS)
7560 {
7561 if ( !pVCpu->hm.s.fSingleInstruction
7562 && !DBGFIsStepping(pVCpu))
7563 {
7564 /*
7565 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7566 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7567 * See Intel spec. 27.3.4 "Saving Non-Register State".
7568 */
7569 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7570 AssertRCReturn(rc2, rc2);
7571 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7572 }
7573 else if (pMixedCtx->eflags.Bits.u1TF)
7574 {
7575 /*
7576 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7577 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7578 */
7579 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7580 uIntrState = 0;
7581 }
7582 }
7583
7584 /*
7585 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7586 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7587 */
7588 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7589 AssertRC(rc2);
7590
7591 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7592 NOREF(fBlockMovSS); NOREF(fBlockSti);
7593 return rc;
7594}
7595
7596
7597/**
7598 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7599 *
7600 * @param pVCpu Pointer to the VMCPU.
7601 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7602 * out-of-sync. Make sure to update the required fields
7603 * before using them.
7604 */
7605DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7606{
7607 NOREF(pMixedCtx);
7608 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7609 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7610}
7611
7612
7613/**
7614 * Injects a double-fault (#DF) exception into the VM.
7615 *
7616 * @returns VBox status code (informational status code included).
7617 * @param pVCpu Pointer to the VMCPU.
7618 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7619 * out-of-sync. Make sure to update the required fields
7620 * before using them.
7621 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7622 * and should return VINF_EM_DBG_STEPPED if the event
7623 * is injected directly (register modified by us, not
7624 * by hardware on VM-entry).
7625 * @param puIntrState Pointer to the current guest interruptibility-state.
7626 * This interruptibility-state will be updated if
7627 * necessary. This cannot not be NULL.
7628 */
7629DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7630{
7631 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7632 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7633 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7634 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7635 fStepping, puIntrState);
7636}
7637
7638
7639/**
7640 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7641 *
7642 * @param pVCpu Pointer to the VMCPU.
7643 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7644 * out-of-sync. Make sure to update the required fields
7645 * before using them.
7646 */
7647DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7648{
7649 NOREF(pMixedCtx);
7650 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7651 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7652 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7653}
7654
7655
7656/**
7657 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7658 *
7659 * @param pVCpu Pointer to the VMCPU.
7660 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7661 * out-of-sync. Make sure to update the required fields
7662 * before using them.
7663 * @param cbInstr The value of RIP that is to be pushed on the guest
7664 * stack.
7665 */
7666DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7667{
7668 NOREF(pMixedCtx);
7669 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7670 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7671 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7672}
7673
7674
7675/**
7676 * Injects a general-protection (#GP) fault into the VM.
7677 *
7678 * @returns VBox status code (informational status code included).
7679 * @param pVCpu Pointer to the VMCPU.
7680 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7681 * out-of-sync. Make sure to update the required fields
7682 * before using them.
7683 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7684 * mode, i.e. in real-mode it's not valid).
7685 * @param u32ErrorCode The error code associated with the #GP.
7686 * @param fStepping Whether we're running in
7687 * hmR0VmxRunGuestCodeStep() and should return
7688 * VINF_EM_DBG_STEPPED if the event is injected
7689 * directly (register modified by us, not by
7690 * hardware on VM-entry).
7691 * @param puIntrState Pointer to the current guest interruptibility-state.
7692 * This interruptibility-state will be updated if
7693 * necessary. This cannot not be NULL.
7694 */
7695DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7696 bool fStepping, uint32_t *puIntrState)
7697{
7698 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7699 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7700 if (fErrorCodeValid)
7701 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7702 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7703 fStepping, puIntrState);
7704}
7705
7706
7707/**
7708 * Sets a general-protection (#GP) exception as pending-for-injection into the
7709 * VM.
7710 *
7711 * @param pVCpu Pointer to the VMCPU.
7712 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7713 * out-of-sync. Make sure to update the required fields
7714 * before using them.
7715 * @param u32ErrorCode The error code associated with the #GP.
7716 */
7717DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7718{
7719 NOREF(pMixedCtx);
7720 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7721 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7722 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7723 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7724}
7725
7726
7727/**
7728 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7729 *
7730 * @param pVCpu Pointer to the VMCPU.
7731 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7732 * out-of-sync. Make sure to update the required fields
7733 * before using them.
7734 * @param uVector The software interrupt vector number.
7735 * @param cbInstr The value of RIP that is to be pushed on the guest
7736 * stack.
7737 */
7738DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7739{
7740 NOREF(pMixedCtx);
7741 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7742 if ( uVector == X86_XCPT_BP
7743 || uVector == X86_XCPT_OF)
7744 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7745 else
7746 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7747 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7748}
7749
7750
7751/**
7752 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7753 * stack.
7754 *
7755 * @returns VBox status code (information status code included).
7756 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7757 * @param pVM Pointer to the VM.
7758 * @param pMixedCtx Pointer to the guest-CPU context.
7759 * @param uValue The value to push to the guest stack.
7760 */
7761DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7762{
7763 /*
7764 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7765 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7766 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7767 */
7768 if (pMixedCtx->sp == 1)
7769 return VINF_EM_RESET;
7770 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7771 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7772 AssertRCReturn(rc, rc);
7773 return rc;
7774}
7775
7776
7777/**
7778 * Injects an event into the guest upon VM-entry by updating the relevant fields
7779 * in the VM-entry area in the VMCS.
7780 *
7781 * @returns VBox status code (informational error codes included).
7782 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7783 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7784 *
7785 * @param pVCpu Pointer to the VMCPU.
7786 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7787 * be out-of-sync. Make sure to update the required
7788 * fields before using them.
7789 * @param u64IntInfo The VM-entry interruption-information field.
7790 * @param cbInstr The VM-entry instruction length in bytes (for
7791 * software interrupts, exceptions and privileged
7792 * software exceptions).
7793 * @param u32ErrCode The VM-entry exception error code.
7794 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7795 * @param puIntrState Pointer to the current guest interruptibility-state.
7796 * This interruptibility-state will be updated if
7797 * necessary. This cannot not be NULL.
7798 * @param fStepping Whether we're running in
7799 * hmR0VmxRunGuestCodeStep() and should return
7800 * VINF_EM_DBG_STEPPED if the event is injected
7801 * directly (register modified by us, not by
7802 * hardware on VM-entry).
7803 *
7804 * @remarks Requires CR0!
7805 * @remarks No-long-jump zone!!!
7806 */
7807static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7808 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *puIntrState)
7809{
7810 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7811 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7812 Assert(puIntrState);
7813 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7814
7815 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7816 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7817
7818#ifdef VBOX_STRICT
7819 /* Validate the error-code-valid bit for hardware exceptions. */
7820 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7821 {
7822 switch (uVector)
7823 {
7824 case X86_XCPT_PF:
7825 case X86_XCPT_DF:
7826 case X86_XCPT_TS:
7827 case X86_XCPT_NP:
7828 case X86_XCPT_SS:
7829 case X86_XCPT_GP:
7830 case X86_XCPT_AC:
7831 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7832 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7833 /* fallthru */
7834 default:
7835 break;
7836 }
7837 }
7838#endif
7839
7840 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7841 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7842 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7843
7844 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7845
7846 /* We require CR0 to check if the guest is in real-mode. */
7847 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7848 AssertRCReturn(rc, rc);
7849
7850 /*
7851 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7852 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7853 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7854 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7855 */
7856 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7857 {
7858 PVM pVM = pVCpu->CTX_SUFF(pVM);
7859 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7860 {
7861 Assert(PDMVmmDevHeapIsEnabled(pVM));
7862 Assert(pVM->hm.s.vmx.pRealModeTSS);
7863
7864 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7865 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7866 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7867 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7868 AssertRCReturn(rc, rc);
7869 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7870
7871 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7872 size_t const cbIdtEntry = sizeof(X86IDTR16);
7873 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7874 {
7875 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7876 if (uVector == X86_XCPT_DF)
7877 return VINF_EM_RESET;
7878
7879 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7880 if (uVector == X86_XCPT_GP)
7881 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7882
7883 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7884 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7885 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7886 fStepping, puIntrState);
7887 }
7888
7889 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7890 uint16_t uGuestIp = pMixedCtx->ip;
7891 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7892 {
7893 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7894 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7895 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7896 }
7897 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7898 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7899
7900 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7901 X86IDTR16 IdtEntry;
7902 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7903 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7904 AssertRCReturn(rc, rc);
7905
7906 /* Construct the stack frame for the interrupt/exception handler. */
7907 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7908 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7909 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7910 AssertRCReturn(rc, rc);
7911
7912 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7913 if (rc == VINF_SUCCESS)
7914 {
7915 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7916 pMixedCtx->rip = IdtEntry.offSel;
7917 pMixedCtx->cs.Sel = IdtEntry.uSel;
7918 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7919 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7920 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7921 && uVector == X86_XCPT_PF)
7922 pMixedCtx->cr2 = GCPtrFaultAddress;
7923
7924 /* If any other guest-state bits are changed here, make sure to update
7925 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7926 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7927 | HM_CHANGED_GUEST_RIP
7928 | HM_CHANGED_GUEST_RFLAGS
7929 | HM_CHANGED_GUEST_RSP);
7930
7931 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7932 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7933 {
7934 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7935 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7936 Log4(("Clearing inhibition due to STI.\n"));
7937 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7938 }
7939 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7940 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7941
7942 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7943 it, if we are returning to ring-3 before executing guest code. */
7944 pVCpu->hm.s.Event.fPending = false;
7945
7946 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7947 if (fStepping)
7948 rc = VINF_EM_DBG_STEPPED;
7949 }
7950 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
7951 return rc;
7952 }
7953
7954 /*
7955 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7956 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7957 */
7958 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7959 }
7960
7961 /* Validate. */
7962 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7963 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7964 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7965
7966 /* Inject. */
7967 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7968 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7969 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7970 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7971
7972 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7973 && uVector == X86_XCPT_PF)
7974 pMixedCtx->cr2 = GCPtrFaultAddress;
7975
7976 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7977 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7978
7979 AssertRCReturn(rc, rc);
7980 return rc;
7981}
7982
7983
7984/**
7985 * Clears the interrupt-window exiting control in the VMCS and if necessary
7986 * clears the current event in the VMCS as well.
7987 *
7988 * @returns VBox status code.
7989 * @param pVCpu Pointer to the VMCPU.
7990 *
7991 * @remarks Use this function only to clear events that have not yet been
7992 * delivered to the guest but are injected in the VMCS!
7993 * @remarks No-long-jump zone!!!
7994 */
7995static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7996{
7997 int rc;
7998 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7999
8000 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8001 {
8002 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8003 Assert(!pVCpu->hm.s.Event.fPending);
8004 }
8005
8006 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8007 {
8008 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8009 Assert(!pVCpu->hm.s.Event.fPending);
8010 }
8011
8012 if (!pVCpu->hm.s.Event.fPending)
8013 return;
8014
8015#ifdef VBOX_STRICT
8016 uint32_t u32EntryInfo;
8017 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8018 AssertRC(rc);
8019 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
8020#endif
8021
8022 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8023 AssertRC(rc);
8024
8025 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8026 AssertRC(rc);
8027
8028 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8029 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8030}
8031
8032
8033/**
8034 * Enters the VT-x session.
8035 *
8036 * @returns VBox status code.
8037 * @param pVM Pointer to the VM.
8038 * @param pVCpu Pointer to the VMCPU.
8039 * @param pCpu Pointer to the CPU info struct.
8040 */
8041VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8042{
8043 AssertPtr(pVM);
8044 AssertPtr(pVCpu);
8045 Assert(pVM->hm.s.vmx.fSupported);
8046 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8047 NOREF(pCpu); NOREF(pVM);
8048
8049 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8050 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8051
8052#ifdef VBOX_STRICT
8053 /* Make sure we're in VMX root mode. */
8054 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8055 if (!(u32HostCR4 & X86_CR4_VMXE))
8056 {
8057 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8058 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8059 }
8060#endif
8061
8062 /*
8063 * Load the VCPU's VMCS as the current (and active) one.
8064 */
8065 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8066 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8067 if (RT_FAILURE(rc))
8068 return rc;
8069
8070 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8071 pVCpu->hm.s.fLeaveDone = false;
8072 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8073
8074 return VINF_SUCCESS;
8075}
8076
8077
8078/**
8079 * The thread-context callback (only on platforms which support it).
8080 *
8081 * @param enmEvent The thread-context event.
8082 * @param pVCpu Pointer to the VMCPU.
8083 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8084 * @thread EMT(pVCpu)
8085 */
8086VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8087{
8088 NOREF(fGlobalInit);
8089
8090 switch (enmEvent)
8091 {
8092 case RTTHREADCTXEVENT_PREEMPTING:
8093 {
8094 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8095 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8096 VMCPU_ASSERT_EMT(pVCpu);
8097
8098 PVM pVM = pVCpu->CTX_SUFF(pVM);
8099 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8100
8101 /* No longjmps (logger flushes, locks) in this fragile context. */
8102 VMMRZCallRing3Disable(pVCpu);
8103 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8104
8105 /*
8106 * Restore host-state (FPU, debug etc.)
8107 */
8108 if (!pVCpu->hm.s.fLeaveDone)
8109 {
8110 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8111 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8112 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8113 pVCpu->hm.s.fLeaveDone = true;
8114 }
8115
8116 /* Leave HM context, takes care of local init (term). */
8117 int rc = HMR0LeaveCpu(pVCpu);
8118 AssertRC(rc); NOREF(rc);
8119
8120 /* Restore longjmp state. */
8121 VMMRZCallRing3Enable(pVCpu);
8122 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8123 break;
8124 }
8125
8126 case RTTHREADCTXEVENT_RESUMED:
8127 {
8128 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8129 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8130 VMCPU_ASSERT_EMT(pVCpu);
8131
8132 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8133 VMMRZCallRing3Disable(pVCpu);
8134 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8135
8136 /* Initialize the bare minimum state required for HM. This takes care of
8137 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8138 int rc = HMR0EnterCpu(pVCpu);
8139 AssertRC(rc);
8140 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8141
8142 /* Load the active VMCS as the current one. */
8143 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8144 {
8145 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8146 AssertRC(rc); NOREF(rc);
8147 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8148 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8149 }
8150 pVCpu->hm.s.fLeaveDone = false;
8151
8152 /* Restore longjmp state. */
8153 VMMRZCallRing3Enable(pVCpu);
8154 break;
8155 }
8156
8157 default:
8158 break;
8159 }
8160}
8161
8162
8163/**
8164 * Saves the host state in the VMCS host-state.
8165 * Sets up the VM-exit MSR-load area.
8166 *
8167 * The CPU state will be loaded from these fields on every successful VM-exit.
8168 *
8169 * @returns VBox status code.
8170 * @param pVM Pointer to the VM.
8171 * @param pVCpu Pointer to the VMCPU.
8172 *
8173 * @remarks No-long-jump zone!!!
8174 */
8175static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8176{
8177 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8178
8179 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8180 return VINF_SUCCESS;
8181
8182 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8183 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8184
8185 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8186 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8187
8188 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8189 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8190
8191 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8192 return rc;
8193}
8194
8195
8196/**
8197 * Saves the host state in the VMCS host-state.
8198 *
8199 * @returns VBox status code.
8200 * @param pVM Pointer to the VM.
8201 * @param pVCpu Pointer to the VMCPU.
8202 *
8203 * @remarks No-long-jump zone!!!
8204 */
8205VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8206{
8207 AssertPtr(pVM);
8208 AssertPtr(pVCpu);
8209
8210 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8211
8212 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8213 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8214 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8215 return hmR0VmxSaveHostState(pVM, pVCpu);
8216}
8217
8218
8219/**
8220 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8221 * loaded from these fields on every successful VM-entry.
8222 *
8223 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8224 * Sets up the VM-entry controls.
8225 * Sets up the appropriate VMX non-root function to execute guest code based on
8226 * the guest CPU mode.
8227 *
8228 * @returns VBox status code.
8229 * @param pVM Pointer to the VM.
8230 * @param pVCpu Pointer to the VMCPU.
8231 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8232 * out-of-sync. Make sure to update the required fields
8233 * before using them.
8234 *
8235 * @remarks No-long-jump zone!!!
8236 */
8237static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8238{
8239 AssertPtr(pVM);
8240 AssertPtr(pVCpu);
8241 AssertPtr(pMixedCtx);
8242 HMVMX_ASSERT_PREEMPT_SAFE();
8243
8244 VMMRZCallRing3Disable(pVCpu);
8245 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8246
8247 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8248
8249 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8250
8251 /* Determine real-on-v86 mode. */
8252 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8253 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8254 && CPUMIsGuestInRealModeEx(pMixedCtx))
8255 {
8256 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8257 }
8258
8259 /*
8260 * Load the guest-state into the VMCS.
8261 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8262 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8263 */
8264 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8265 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8266
8267 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8268 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8269 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8270
8271 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8272 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8273 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8274
8275 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8276 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8277
8278 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8279 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8280
8281 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8282 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8283 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8284
8285 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8286 determine we don't have to swap EFER after all. */
8287 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8288 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8289
8290 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8291 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8292
8293 /*
8294 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8295 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8296 */
8297 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8298 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8299
8300 /* Clear any unused and reserved bits. */
8301 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8302
8303 VMMRZCallRing3Enable(pVCpu);
8304
8305 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8306 return rc;
8307}
8308
8309
8310/**
8311 * Loads the state shared between the host and guest into the VMCS.
8312 *
8313 * @param pVM Pointer to the VM.
8314 * @param pVCpu Pointer to the VMCPU.
8315 * @param pCtx Pointer to the guest-CPU context.
8316 *
8317 * @remarks No-long-jump zone!!!
8318 */
8319static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8320{
8321 NOREF(pVM);
8322
8323 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8324 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8325
8326 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8327 {
8328 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8329 AssertRC(rc);
8330 }
8331
8332 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8333 {
8334 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8335 AssertRC(rc);
8336
8337 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8338 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8339 {
8340 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8341 AssertRC(rc);
8342 }
8343 }
8344
8345 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8346 {
8347#if HC_ARCH_BITS == 64
8348 if (pVM->hm.s.fAllow64BitGuests)
8349 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8350#endif
8351 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8352 }
8353
8354 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8355 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8356}
8357
8358
8359/**
8360 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8361 *
8362 * @param pVM Pointer to the VM.
8363 * @param pVCpu Pointer to the VMCPU.
8364 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8365 * out-of-sync. Make sure to update the required fields
8366 * before using them.
8367 */
8368DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8369{
8370 HMVMX_ASSERT_PREEMPT_SAFE();
8371
8372 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8373#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8374 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8375#endif
8376
8377 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8378 {
8379 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8380 AssertRC(rc);
8381 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8382 }
8383 else if (HMCPU_CF_VALUE(pVCpu))
8384 {
8385 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8386 AssertRC(rc);
8387 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8388 }
8389
8390 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8391 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8392 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8393 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8394}
8395
8396
8397/**
8398 * Does the preparations before executing guest code in VT-x.
8399 *
8400 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8401 * recompiler/IEM. We must be cautious what we do here regarding committing
8402 * guest-state information into the VMCS assuming we assuredly execute the
8403 * guest in VT-x mode.
8404 *
8405 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8406 * the common-state (TRPM/forceflags), we must undo those changes so that the
8407 * recompiler/IEM can (and should) use them when it resumes guest execution.
8408 * Otherwise such operations must be done when we can no longer exit to ring-3.
8409 *
8410 * @returns Strict VBox status code.
8411 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8412 * have been disabled.
8413 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8414 * double-fault into the guest.
8415 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8416 * dispatched directly.
8417 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8418 *
8419 * @param pVM Pointer to the VM.
8420 * @param pVCpu Pointer to the VMCPU.
8421 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8422 * out-of-sync. Make sure to update the required fields
8423 * before using them.
8424 * @param pVmxTransient Pointer to the VMX transient structure.
8425 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8426 * us ignore some of the reasons for returning to
8427 * ring-3, and return VINF_EM_DBG_STEPPED if event
8428 * dispatching took place.
8429 */
8430static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8431{
8432 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8433
8434#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8435 PGMRZDynMapFlushAutoSet(pVCpu);
8436#endif
8437
8438 /* Check force flag actions that might require us to go back to ring-3. */
8439 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8440 if (rc != VINF_SUCCESS)
8441 return rc;
8442
8443#ifndef IEM_VERIFICATION_MODE_FULL
8444 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8445 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8446 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8447 {
8448 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8449 RTGCPHYS GCPhysApicBase;
8450 GCPhysApicBase = pMixedCtx->msrApicBase;
8451 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8452
8453 /* Unalias any existing mapping. */
8454 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8455 AssertRCReturn(rc, rc);
8456
8457 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8458 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8459 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8460 AssertRCReturn(rc, rc);
8461
8462 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8463 }
8464#endif /* !IEM_VERIFICATION_MODE_FULL */
8465
8466 if (TRPMHasTrap(pVCpu))
8467 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8468 else if (!pVCpu->hm.s.Event.fPending)
8469 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8470
8471 /*
8472 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8473 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8474 */
8475 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fStepping);
8476 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8477 {
8478 Assert(rc == VINF_EM_RESET || (rc == VINF_EM_DBG_STEPPED && fStepping));
8479 return rc;
8480 }
8481
8482 /*
8483 * Load the guest state bits, we can handle longjmps/getting preempted here.
8484 *
8485 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8486 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8487 * Hence, this needs to be done -after- injection of events.
8488 */
8489 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8490
8491 /*
8492 * No longjmps to ring-3 from this point on!!!
8493 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8494 * This also disables flushing of the R0-logger instance (if any).
8495 */
8496 VMMRZCallRing3Disable(pVCpu);
8497
8498 /*
8499 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8500 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8501 *
8502 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8503 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8504 *
8505 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8506 * executing guest code.
8507 */
8508 pVmxTransient->uEflags = ASMIntDisableFlags();
8509 if ( ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8510 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8511 && ( !fStepping /* Optimized for the non-stepping case, of course. */
8512 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8513 {
8514 hmR0VmxClearEventVmcs(pVCpu);
8515 ASMSetFlags(pVmxTransient->uEflags);
8516 VMMRZCallRing3Enable(pVCpu);
8517 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8518 return VINF_EM_RAW_TO_R3;
8519 }
8520
8521 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8522 {
8523 hmR0VmxClearEventVmcs(pVCpu);
8524 ASMSetFlags(pVmxTransient->uEflags);
8525 VMMRZCallRing3Enable(pVCpu);
8526 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8527 return VINF_EM_RAW_INTERRUPT;
8528 }
8529
8530 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8531 pVCpu->hm.s.Event.fPending = false;
8532
8533 return VINF_SUCCESS;
8534}
8535
8536
8537/**
8538 * Prepares to run guest code in VT-x and we've committed to doing so. This
8539 * means there is no backing out to ring-3 or anywhere else at this
8540 * point.
8541 *
8542 * @param pVM Pointer to the VM.
8543 * @param pVCpu Pointer to the VMCPU.
8544 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8545 * out-of-sync. Make sure to update the required fields
8546 * before using them.
8547 * @param pVmxTransient Pointer to the VMX transient structure.
8548 *
8549 * @remarks Called with preemption disabled.
8550 * @remarks No-long-jump zone!!!
8551 */
8552static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8553{
8554 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8555 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8556 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8557
8558 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8559 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8560
8561#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8562 if (!CPUMIsGuestFPUStateActive(pVCpu))
8563 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8564 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8565#endif
8566
8567 if ( pVCpu->hm.s.fUseGuestFpu
8568 && !CPUMIsGuestFPUStateActive(pVCpu))
8569 {
8570 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8571 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8572 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8573 }
8574
8575 /*
8576 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8577 */
8578 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8579 && pVCpu->hm.s.vmx.cMsrs > 0)
8580 {
8581 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8582 }
8583
8584 /*
8585 * Load the host state bits as we may've been preempted (only happens when
8586 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8587 */
8588 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8589 {
8590 /* This ASSUMES that pfnStartVM has been set up already. */
8591 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8592 AssertRC(rc);
8593 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8594 }
8595 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8596
8597 /*
8598 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8599 */
8600 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8601 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8602 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8603
8604 /* Store status of the shared guest-host state at the time of VM-entry. */
8605#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8606 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8607 {
8608 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8609 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8610 }
8611 else
8612#endif
8613 {
8614 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8615 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8616 }
8617 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8618
8619 /*
8620 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8621 */
8622 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8623 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8624
8625 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8626 RTCPUID idCurrentCpu = pCpu->idCpu;
8627 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8628 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8629 {
8630 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8631 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8632 }
8633
8634 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8635 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8636 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8637 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8638
8639 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8640
8641 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8642 to start executing. */
8643
8644 /*
8645 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8646 */
8647 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8648 {
8649 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8650 {
8651 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8652 AssertRC(rc2);
8653 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8654 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8655 true /* fUpdateHostMsr */);
8656 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8657 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8658 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8659 }
8660 else
8661 {
8662 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8663 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8664 }
8665 }
8666
8667#ifdef VBOX_STRICT
8668 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8669 hmR0VmxCheckHostEferMsr(pVCpu);
8670 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8671#endif
8672#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8673 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8674 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8675 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8676#endif
8677}
8678
8679
8680/**
8681 * Performs some essential restoration of state after running guest code in
8682 * VT-x.
8683 *
8684 * @param pVM Pointer to the VM.
8685 * @param pVCpu Pointer to the VMCPU.
8686 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8687 * out-of-sync. Make sure to update the required fields
8688 * before using them.
8689 * @param pVmxTransient Pointer to the VMX transient structure.
8690 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8691 *
8692 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8693 *
8694 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8695 * unconditionally when it is safe to do so.
8696 */
8697static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8698{
8699 NOREF(pVM);
8700
8701 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8702
8703 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8704 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8705 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8706 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8707 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8708 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8709
8710 /** @todo Last-seen-tick shouldn't be necessary when TM supports invariant
8711 * mode. */
8712 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8713 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8714
8715 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8716 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8717 Assert(!(ASMGetFlags() & X86_EFL_IF));
8718 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8719
8720#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8721 if (CPUMIsGuestFPUStateActive(pVCpu))
8722 {
8723 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8724 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8725 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8726 }
8727#endif
8728
8729#if HC_ARCH_BITS == 64
8730 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8731#endif
8732 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8733#ifdef VBOX_STRICT
8734 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8735#endif
8736 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8737 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8738
8739 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8740 uint32_t uExitReason;
8741 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8742 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8743 AssertRC(rc);
8744 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8745 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8746
8747 /* Update the VM-exit history array. */
8748 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8749
8750 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8751 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8752 {
8753 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8754 pVmxTransient->fVMEntryFailed));
8755 return;
8756 }
8757
8758 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8759 {
8760 /** @todo We can optimize this by only syncing with our force-flags when
8761 * really needed and keeping the VMCS state as it is for most
8762 * VM-exits. */
8763 /* Update the guest interruptibility-state from the VMCS. */
8764 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8765
8766#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8767 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8768 AssertRC(rc);
8769#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8770 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8771 AssertRC(rc);
8772#endif
8773
8774 /*
8775 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8776 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8777 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8778 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8779 */
8780 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8781 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8782 {
8783 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8784 AssertRC(rc);
8785 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8786 }
8787 }
8788}
8789
8790
8791/**
8792 * Runs the guest code using VT-x the normal way.
8793 *
8794 * @returns VBox status code.
8795 * @param pVM Pointer to the VM.
8796 * @param pVCpu Pointer to the VMCPU.
8797 * @param pCtx Pointer to the guest-CPU context.
8798 *
8799 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8800 */
8801static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8802{
8803 VMXTRANSIENT VmxTransient;
8804 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8805 int rc = VERR_INTERNAL_ERROR_5;
8806 uint32_t cLoops = 0;
8807
8808 for (;; cLoops++)
8809 {
8810 Assert(!HMR0SuspendPending());
8811 HMVMX_ASSERT_CPU_SAFE();
8812
8813 /* Preparatory work for running guest code, this may force us to return
8814 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8815 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8816 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8817 if (rc != VINF_SUCCESS)
8818 break;
8819
8820 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8821 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8822 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8823
8824 /* Restore any residual host-state and save any bits shared between host
8825 and guest into the guest-CPU state. Re-enables interrupts! */
8826 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8827
8828 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8829 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8830 {
8831 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8832 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8833 return rc;
8834 }
8835
8836 /* Profiling the VM-exit. */
8837 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8838 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8839 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8840 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8841 HMVMX_START_EXIT_DISPATCH_PROF();
8842
8843 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8844 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8845 {
8846 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8847 hmR0VmxSaveGuestState(pVCpu, pCtx);
8848 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8849 }
8850
8851 /* Handle the VM-exit. */
8852#ifdef HMVMX_USE_FUNCTION_TABLE
8853 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8854#else
8855 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8856#endif
8857 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8858 if (rc != VINF_SUCCESS)
8859 break;
8860 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8861 {
8862 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8863 rc = VINF_EM_RAW_INTERRUPT;
8864 break;
8865 }
8866 }
8867
8868 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8869 return rc;
8870}
8871
8872
8873/**
8874 * Single steps guest code using VT-x.
8875 *
8876 * @returns VBox status code.
8877 * @param pVM Pointer to the VM.
8878 * @param pVCpu Pointer to the VMCPU.
8879 * @param pCtx Pointer to the guest-CPU context.
8880 *
8881 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8882 */
8883static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8884{
8885 VMXTRANSIENT VmxTransient;
8886 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8887 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8888 uint32_t cLoops = 0;
8889 uint16_t uCsStart = pCtx->cs.Sel;
8890 uint64_t uRipStart = pCtx->rip;
8891
8892 for (;; cLoops++)
8893 {
8894 Assert(!HMR0SuspendPending());
8895 HMVMX_ASSERT_CPU_SAFE();
8896
8897 /* Preparatory work for running guest code, this may force us to return
8898 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8899 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8900 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, true /* fStepping */);
8901 if (rcStrict != VINF_SUCCESS)
8902 break;
8903
8904 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8905 rcStrict = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8906 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8907
8908 /* Restore any residual host-state and save any bits shared between host
8909 and guest into the guest-CPU state. Re-enables interrupts! */
8910 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8911
8912 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8913 if (RT_UNLIKELY(rcStrict != VINF_SUCCESS))
8914 {
8915 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8916 hmR0VmxReportWorldSwitchError(pVM, pVCpu, VBOXSTRICTRC_TODO(rcStrict), pCtx, &VmxTransient);
8917 return VBOXSTRICTRC_TODO(rcStrict);
8918 }
8919 /* Profiling the VM-exit. */
8920 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8921 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8922 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8923 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8924 HMVMX_START_EXIT_DISPATCH_PROF();
8925
8926 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8927 if (RT_UNLIKELY(VBOXVMM_R0_HMVMX_VMEXIT_ENABLED()))
8928 {
8929 hmR0VmxReadExitQualificationVmcs(pVCpu, &VmxTransient);
8930 hmR0VmxSaveGuestState(pVCpu, pCtx);
8931 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, VmxTransient.uExitReason, VmxTransient.uExitQualification);
8932 }
8933
8934 /* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitStep(). */
8935 rcStrict = hmR0VmxHandleExitStep(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, uCsStart, uRipStart);
8936 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8937 if (rcStrict != VINF_SUCCESS)
8938 break;
8939 if (cLoops > pVM->hm.s.cMaxResumeLoops)
8940 {
8941 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8942 rcStrict = VINF_EM_RAW_INTERRUPT;
8943 break;
8944 }
8945
8946 /*
8947 * Did the RIP change, if so, consider it a single step.
8948 * Otherwise, make sure one of the TFs gets set.
8949 */
8950 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8951 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8952 AssertRCReturn(rc2, rc2);
8953 if ( pCtx->rip != uRipStart
8954 || pCtx->cs.Sel != uCsStart)
8955 {
8956 rcStrict = VINF_EM_DBG_STEPPED;
8957 break;
8958 }
8959 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8960 }
8961
8962 /*
8963 * Clear the X86_EFL_TF if necessary.
8964 */
8965 if (pVCpu->hm.s.fClearTrapFlag)
8966 {
8967 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8968 AssertRCReturn(rc2, rc2);
8969 pVCpu->hm.s.fClearTrapFlag = false;
8970 pCtx->eflags.Bits.u1TF = 0;
8971 }
8972 /** @todo there seems to be issues with the resume flag when the monitor trap
8973 * flag is pending without being used. Seen early in bios init when
8974 * accessing APIC page in protected mode. */
8975
8976 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8977 return VBOXSTRICTRC_TODO(rcStrict);
8978}
8979
8980
8981/**
8982 * Runs the guest code using VT-x.
8983 *
8984 * @returns VBox status code.
8985 * @param pVM Pointer to the VM.
8986 * @param pVCpu Pointer to the VMCPU.
8987 * @param pCtx Pointer to the guest-CPU context.
8988 */
8989VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8990{
8991 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8992 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8993 HMVMX_ASSERT_PREEMPT_SAFE();
8994
8995 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8996
8997 int rc;
8998 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8999 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
9000 else
9001 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
9002
9003 if (rc == VERR_EM_INTERPRETER)
9004 rc = VINF_EM_RAW_EMULATE_INSTR;
9005 else if (rc == VINF_EM_RESET)
9006 rc = VINF_EM_TRIPLE_FAULT;
9007
9008 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
9009 if (RT_FAILURE(rc2))
9010 {
9011 pVCpu->hm.s.u32HMError = rc;
9012 rc = rc2;
9013 }
9014 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
9015 return rc;
9016}
9017
9018
9019#ifndef HMVMX_USE_FUNCTION_TABLE
9020DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
9021{
9022#ifdef DEBUG_ramshankar
9023# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
9024# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
9025#endif
9026 int rc;
9027 switch (rcReason)
9028 {
9029 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9030 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9031 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9032 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9033 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9034 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9035 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9036 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9037 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9038 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9039 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9040 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9041 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9042 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9043 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9044 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9045 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9046 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9047 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9048 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9049 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9050 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9051 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9052 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9053 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9054 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9055 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9056 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9057 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9058 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9059 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9060 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9061 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9062 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9063
9064 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9065 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9066 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9067 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9068 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9069 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9070 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9071 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9072 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9073
9074 case VMX_EXIT_VMCLEAR:
9075 case VMX_EXIT_VMLAUNCH:
9076 case VMX_EXIT_VMPTRLD:
9077 case VMX_EXIT_VMPTRST:
9078 case VMX_EXIT_VMREAD:
9079 case VMX_EXIT_VMRESUME:
9080 case VMX_EXIT_VMWRITE:
9081 case VMX_EXIT_VMXOFF:
9082 case VMX_EXIT_VMXON:
9083 case VMX_EXIT_INVEPT:
9084 case VMX_EXIT_INVVPID:
9085 case VMX_EXIT_VMFUNC:
9086 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9087 break;
9088 default:
9089 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9090 break;
9091 }
9092 return rc;
9093}
9094#endif /* !HMVMX_USE_FUNCTION_TABLE */
9095
9096
9097/**
9098 * Single-stepping VM-exit filtering.
9099 *
9100 * This is preprocessing the exits and deciding whether we've gotten far enough
9101 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9102 * performed.
9103 *
9104 * @returns Strict VBox status code.
9105 * @param pVCpu The virtual CPU of the calling EMT.
9106 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9107 * out-of-sync. Make sure to update the required
9108 * fields before using them.
9109 * @param pVmxTransient Pointer to the VMX-transient structure.
9110 * @param uExitReason The VM-exit reason.
9111 */
9112DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitStep(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9113 uint32_t uExitReason, uint16_t uCsStart, uint64_t uRipStart)
9114{
9115 switch (uExitReason)
9116 {
9117 case VMX_EXIT_XCPT_OR_NMI:
9118 {
9119 /* Check for host NMI. */
9120 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9121 AssertRCReturn(rc2, rc2);
9122 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9123 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9124 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9125 /* fall thru */
9126 }
9127
9128 case VMX_EXIT_EPT_MISCONFIG:
9129 case VMX_EXIT_TRIPLE_FAULT:
9130 case VMX_EXIT_APIC_ACCESS:
9131 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9132 case VMX_EXIT_TASK_SWITCH:
9133
9134 /* Instruction specific VM-exits: */
9135 case VMX_EXIT_IO_INSTR:
9136 case VMX_EXIT_CPUID:
9137 case VMX_EXIT_RDTSC:
9138 case VMX_EXIT_RDTSCP:
9139 case VMX_EXIT_MOV_CRX:
9140 case VMX_EXIT_MWAIT:
9141 case VMX_EXIT_MONITOR:
9142 case VMX_EXIT_RDMSR:
9143 case VMX_EXIT_WRMSR:
9144 case VMX_EXIT_MOV_DRX:
9145 case VMX_EXIT_HLT:
9146 case VMX_EXIT_INVD:
9147 case VMX_EXIT_INVLPG:
9148 case VMX_EXIT_RSM:
9149 case VMX_EXIT_PAUSE:
9150 case VMX_EXIT_XDTR_ACCESS:
9151 case VMX_EXIT_TR_ACCESS:
9152 case VMX_EXIT_WBINVD:
9153 case VMX_EXIT_XSETBV:
9154 case VMX_EXIT_RDRAND:
9155 case VMX_EXIT_INVPCID:
9156 case VMX_EXIT_GETSEC:
9157 case VMX_EXIT_RDPMC:
9158 case VMX_EXIT_VMCALL:
9159 case VMX_EXIT_VMCLEAR:
9160 case VMX_EXIT_VMLAUNCH:
9161 case VMX_EXIT_VMPTRLD:
9162 case VMX_EXIT_VMPTRST:
9163 case VMX_EXIT_VMREAD:
9164 case VMX_EXIT_VMRESUME:
9165 case VMX_EXIT_VMWRITE:
9166 case VMX_EXIT_VMXOFF:
9167 case VMX_EXIT_VMXON:
9168 case VMX_EXIT_INVEPT:
9169 case VMX_EXIT_INVVPID:
9170 case VMX_EXIT_VMFUNC:
9171 {
9172 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9173 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9174 AssertRCReturn(rc2, rc2);
9175 if ( pMixedCtx->rip != uRipStart
9176 || pMixedCtx->cs.Sel != uCsStart)
9177 return VINF_EM_DBG_STEPPED;
9178 break;
9179 }
9180 }
9181
9182 /*
9183 * Normal processing.
9184 */
9185#ifdef HMVMX_USE_FUNCTION_TABLE
9186 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9187#else
9188 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9189#endif
9190}
9191
9192
9193#ifdef DEBUG
9194/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9195# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9196 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9197
9198# define HMVMX_ASSERT_PREEMPT_CPUID() \
9199 do \
9200 { \
9201 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9202 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9203 } while (0)
9204
9205# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9206 do { \
9207 AssertPtr(pVCpu); \
9208 AssertPtr(pMixedCtx); \
9209 AssertPtr(pVmxTransient); \
9210 Assert(pVmxTransient->fVMEntryFailed == false); \
9211 Assert(ASMIntAreEnabled()); \
9212 HMVMX_ASSERT_PREEMPT_SAFE(); \
9213 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9214 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9215 HMVMX_ASSERT_PREEMPT_SAFE(); \
9216 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9217 HMVMX_ASSERT_PREEMPT_CPUID(); \
9218 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9219 } while (0)
9220
9221# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9222 do { \
9223 Log4Func(("\n")); \
9224 } while (0)
9225#else /* Release builds */
9226# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9227 do { \
9228 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9229 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9230 } while (0)
9231# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9232#endif
9233
9234
9235/**
9236 * Advances the guest RIP after reading it from the VMCS.
9237 *
9238 * @returns VBox status code.
9239 * @param pVCpu Pointer to the VMCPU.
9240 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9241 * out-of-sync. Make sure to update the required fields
9242 * before using them.
9243 * @param pVmxTransient Pointer to the VMX transient structure.
9244 *
9245 * @remarks No-long-jump zone!!!
9246 */
9247DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9248{
9249 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9250 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9251 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9252 AssertRCReturn(rc, rc);
9253
9254 pMixedCtx->rip += pVmxTransient->cbInstr;
9255 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9256
9257 /*
9258 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9259 * pending debug exception field as it takes care of priority of events.
9260 *
9261 * See Intel spec. 32.2.1 "Debug Exceptions".
9262 */
9263 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9264
9265 return rc;
9266}
9267
9268
9269/**
9270 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9271 * and update error record fields accordingly.
9272 *
9273 * @return VMX_IGS_* return codes.
9274 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9275 * wrong with the guest state.
9276 *
9277 * @param pVM Pointer to the VM.
9278 * @param pVCpu Pointer to the VMCPU.
9279 * @param pCtx Pointer to the guest-CPU state.
9280 *
9281 * @remarks This function assumes our cache of the VMCS controls
9282 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9283 */
9284static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9285{
9286#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9287#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9288 uError = (err); \
9289 break; \
9290 } else do { } while (0)
9291
9292 int rc;
9293 uint32_t uError = VMX_IGS_ERROR;
9294 uint32_t u32Val;
9295 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9296
9297 do
9298 {
9299 /*
9300 * CR0.
9301 */
9302 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9303 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9304 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9305 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9306 if (fUnrestrictedGuest)
9307 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9308
9309 uint32_t u32GuestCR0;
9310 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9311 AssertRCBreak(rc);
9312 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9313 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9314 if ( !fUnrestrictedGuest
9315 && (u32GuestCR0 & X86_CR0_PG)
9316 && !(u32GuestCR0 & X86_CR0_PE))
9317 {
9318 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9319 }
9320
9321 /*
9322 * CR4.
9323 */
9324 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9325 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9326
9327 uint32_t u32GuestCR4;
9328 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9329 AssertRCBreak(rc);
9330 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9331 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9332
9333 /*
9334 * IA32_DEBUGCTL MSR.
9335 */
9336 uint64_t u64Val;
9337 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9338 AssertRCBreak(rc);
9339 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9340 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9341 {
9342 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9343 }
9344 uint64_t u64DebugCtlMsr = u64Val;
9345
9346#ifdef VBOX_STRICT
9347 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9348 AssertRCBreak(rc);
9349 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9350#endif
9351 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9352
9353 /*
9354 * RIP and RFLAGS.
9355 */
9356 uint32_t u32Eflags;
9357#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9358 if (HMVMX_IS_64BIT_HOST_MODE())
9359 {
9360 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9361 AssertRCBreak(rc);
9362 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9363 if ( !fLongModeGuest
9364 || !pCtx->cs.Attr.n.u1Long)
9365 {
9366 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9367 }
9368 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9369 * must be identical if the "IA-32e mode guest" VM-entry
9370 * control is 1 and CS.L is 1. No check applies if the
9371 * CPU supports 64 linear-address bits. */
9372
9373 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9374 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9375 AssertRCBreak(rc);
9376 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9377 VMX_IGS_RFLAGS_RESERVED);
9378 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9379 u32Eflags = u64Val;
9380 }
9381 else
9382#endif
9383 {
9384 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9385 AssertRCBreak(rc);
9386 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9387 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9388 }
9389
9390 if ( fLongModeGuest
9391 || ( fUnrestrictedGuest
9392 && !(u32GuestCR0 & X86_CR0_PE)))
9393 {
9394 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9395 }
9396
9397 uint32_t u32EntryInfo;
9398 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9399 AssertRCBreak(rc);
9400 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9401 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9402 {
9403 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9404 }
9405
9406 /*
9407 * 64-bit checks.
9408 */
9409#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9410 if (HMVMX_IS_64BIT_HOST_MODE())
9411 {
9412 if ( fLongModeGuest
9413 && !fUnrestrictedGuest)
9414 {
9415 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9416 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9417 }
9418
9419 if ( !fLongModeGuest
9420 && (u32GuestCR4 & X86_CR4_PCIDE))
9421 {
9422 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9423 }
9424
9425 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9426 * 51:32 beyond the processor's physical-address width are 0. */
9427
9428 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9429 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9430 {
9431 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9432 }
9433
9434 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9435 AssertRCBreak(rc);
9436 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9437
9438 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9439 AssertRCBreak(rc);
9440 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9441 }
9442#endif
9443
9444 /*
9445 * PERF_GLOBAL MSR.
9446 */
9447 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9448 {
9449 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9450 AssertRCBreak(rc);
9451 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9452 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9453 }
9454
9455 /*
9456 * PAT MSR.
9457 */
9458 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9459 {
9460 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9461 AssertRCBreak(rc);
9462 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9463 for (unsigned i = 0; i < 8; i++)
9464 {
9465 uint8_t u8Val = (u64Val & 0xff);
9466 if ( u8Val != 0 /* UC */
9467 && u8Val != 1 /* WC */
9468 && u8Val != 4 /* WT */
9469 && u8Val != 5 /* WP */
9470 && u8Val != 6 /* WB */
9471 && u8Val != 7 /* UC- */)
9472 {
9473 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9474 }
9475 u64Val >>= 8;
9476 }
9477 }
9478
9479 /*
9480 * EFER MSR.
9481 */
9482 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9483 {
9484 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9485 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9486 AssertRCBreak(rc);
9487 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9488 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9489 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9490 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9491 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9492 || !(u32GuestCR0 & X86_CR0_PG)
9493 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9494 VMX_IGS_EFER_LMA_LME_MISMATCH);
9495 }
9496
9497 /*
9498 * Segment registers.
9499 */
9500 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9501 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9502 if (!(u32Eflags & X86_EFL_VM))
9503 {
9504 /* CS */
9505 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9506 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9507 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9508 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9509 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9510 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9511 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9512 /* CS cannot be loaded with NULL in protected mode. */
9513 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9514 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9515 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9516 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9517 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9518 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9519 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9520 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9521 else
9522 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9523
9524 /* SS */
9525 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9526 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9527 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9528 if ( !(pCtx->cr0 & X86_CR0_PE)
9529 || pCtx->cs.Attr.n.u4Type == 3)
9530 {
9531 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9532 }
9533 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9534 {
9535 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9536 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9537 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9538 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9539 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9540 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9541 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9542 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9543 }
9544
9545 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9546 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9547 {
9548 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9549 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9550 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9551 || pCtx->ds.Attr.n.u4Type > 11
9552 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9553 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9554 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9555 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9556 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9557 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9558 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9559 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9560 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9561 }
9562 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9563 {
9564 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9565 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9566 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9567 || pCtx->es.Attr.n.u4Type > 11
9568 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9569 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9570 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9571 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9572 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9573 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9574 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9575 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9576 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9577 }
9578 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9579 {
9580 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9581 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9582 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9583 || pCtx->fs.Attr.n.u4Type > 11
9584 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9585 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9586 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9587 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9588 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9589 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9590 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9591 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9592 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9593 }
9594 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9595 {
9596 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9597 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9598 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9599 || pCtx->gs.Attr.n.u4Type > 11
9600 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9601 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9602 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9603 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9604 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9605 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9606 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9607 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9608 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9609 }
9610 /* 64-bit capable CPUs. */
9611#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9612 if (HMVMX_IS_64BIT_HOST_MODE())
9613 {
9614 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9615 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9616 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9617 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9618 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9619 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9620 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9621 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9622 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9623 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9624 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9625 }
9626#endif
9627 }
9628 else
9629 {
9630 /* V86 mode checks. */
9631 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9632 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9633 {
9634 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9635 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9636 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9637 }
9638 else
9639 {
9640 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9641 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9642 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9643 }
9644
9645 /* CS */
9646 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9647 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9648 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9649 /* SS */
9650 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9651 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9652 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9653 /* DS */
9654 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9655 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9656 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9657 /* ES */
9658 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9659 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9660 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9661 /* FS */
9662 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9663 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9664 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9665 /* GS */
9666 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9667 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9668 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9669 /* 64-bit capable CPUs. */
9670#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9671 if (HMVMX_IS_64BIT_HOST_MODE())
9672 {
9673 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9674 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9675 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9676 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9677 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9678 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9679 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9680 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9681 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9682 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9683 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9684 }
9685#endif
9686 }
9687
9688 /*
9689 * TR.
9690 */
9691 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9692 /* 64-bit capable CPUs. */
9693#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9694 if (HMVMX_IS_64BIT_HOST_MODE())
9695 {
9696 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9697 }
9698#endif
9699 if (fLongModeGuest)
9700 {
9701 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9702 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9703 }
9704 else
9705 {
9706 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9707 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9708 VMX_IGS_TR_ATTR_TYPE_INVALID);
9709 }
9710 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9711 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9712 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9713 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9714 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9715 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9716 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9717 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9718
9719 /*
9720 * GDTR and IDTR.
9721 */
9722#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9723 if (HMVMX_IS_64BIT_HOST_MODE())
9724 {
9725 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9726 AssertRCBreak(rc);
9727 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9728
9729 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9730 AssertRCBreak(rc);
9731 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9732 }
9733#endif
9734
9735 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9736 AssertRCBreak(rc);
9737 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9738
9739 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9740 AssertRCBreak(rc);
9741 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9742
9743 /*
9744 * Guest Non-Register State.
9745 */
9746 /* Activity State. */
9747 uint32_t u32ActivityState;
9748 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9749 AssertRCBreak(rc);
9750 HMVMX_CHECK_BREAK( !u32ActivityState
9751 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9752 VMX_IGS_ACTIVITY_STATE_INVALID);
9753 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9754 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9755 uint32_t u32IntrState;
9756 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9757 AssertRCBreak(rc);
9758 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9759 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9760 {
9761 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9762 }
9763
9764 /** @todo Activity state and injecting interrupts. Left as a todo since we
9765 * currently don't use activity states but ACTIVE. */
9766
9767 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9768 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9769
9770 /* Guest interruptibility-state. */
9771 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9772 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9773 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9774 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9775 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9776 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9777 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9778 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9779 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9780 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9781 {
9782 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9783 {
9784 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9785 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9786 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9787 }
9788 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9789 {
9790 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9791 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9792 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9793 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9794 }
9795 }
9796 /** @todo Assumes the processor is not in SMM. */
9797 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9798 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9799 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9800 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9801 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9802 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9803 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9804 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9805 {
9806 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9807 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9808 }
9809
9810 /* Pending debug exceptions. */
9811 if (HMVMX_IS_64BIT_HOST_MODE())
9812 {
9813 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9814 AssertRCBreak(rc);
9815 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9816 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9817 u32Val = u64Val; /* For pending debug exceptions checks below. */
9818 }
9819 else
9820 {
9821 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9822 AssertRCBreak(rc);
9823 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9824 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9825 }
9826
9827 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9828 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9829 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9830 {
9831 if ( (u32Eflags & X86_EFL_TF)
9832 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9833 {
9834 /* Bit 14 is PendingDebug.BS. */
9835 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9836 }
9837 if ( !(u32Eflags & X86_EFL_TF)
9838 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9839 {
9840 /* Bit 14 is PendingDebug.BS. */
9841 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9842 }
9843 }
9844
9845 /* VMCS link pointer. */
9846 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9847 AssertRCBreak(rc);
9848 if (u64Val != UINT64_C(0xffffffffffffffff))
9849 {
9850 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9851 /** @todo Bits beyond the processor's physical-address width MBZ. */
9852 /** @todo 32-bit located in memory referenced by value of this field (as a
9853 * physical address) must contain the processor's VMCS revision ID. */
9854 /** @todo SMM checks. */
9855 }
9856
9857 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9858 * not using Nested Paging? */
9859 if ( pVM->hm.s.fNestedPaging
9860 && !fLongModeGuest
9861 && CPUMIsGuestInPAEModeEx(pCtx))
9862 {
9863 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9864 AssertRCBreak(rc);
9865 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9866
9867 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9868 AssertRCBreak(rc);
9869 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9870
9871 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9872 AssertRCBreak(rc);
9873 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9874
9875 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9876 AssertRCBreak(rc);
9877 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9878 }
9879
9880 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9881 if (uError == VMX_IGS_ERROR)
9882 uError = VMX_IGS_REASON_NOT_FOUND;
9883 } while (0);
9884
9885 pVCpu->hm.s.u32HMError = uError;
9886 return uError;
9887
9888#undef HMVMX_ERROR_BREAK
9889#undef HMVMX_CHECK_BREAK
9890}
9891
9892/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9893/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9894/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9895
9896/** @name VM-exit handlers.
9897 * @{
9898 */
9899
9900/**
9901 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9902 */
9903HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9904{
9905 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9906 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9907 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9908 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9909 return VINF_SUCCESS;
9910 return VINF_EM_RAW_INTERRUPT;
9911}
9912
9913
9914/**
9915 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9916 */
9917HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9918{
9919 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9920 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9921
9922 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9923 AssertRCReturn(rc, rc);
9924
9925 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9926 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9927 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9928 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9929
9930 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9931 {
9932 /*
9933 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9934 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9935 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9936 *
9937 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9938 */
9939 VMXDispatchHostNmi();
9940 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9941 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9942 return VINF_SUCCESS;
9943 }
9944
9945 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9946 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9947 if (RT_UNLIKELY(rc != VINF_SUCCESS))
9948 {
9949 if (rc == VINF_HM_DOUBLE_FAULT)
9950 rc = VINF_SUCCESS;
9951 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9952 return rc;
9953 }
9954
9955 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9956 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9957 switch (uIntType)
9958 {
9959 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9960 Assert(uVector == X86_XCPT_DB);
9961 /* no break */
9962 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9963 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9964 /* no break */
9965 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9966 {
9967 switch (uVector)
9968 {
9969 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9970 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9971 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9972 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9973 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9974 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9975#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9976 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9977 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9978 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9979 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9980 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9981 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9982 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9983 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9984 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9985 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9986 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9987 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9988#endif
9989 default:
9990 {
9991 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9992 AssertRCReturn(rc, rc);
9993
9994 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9995 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9996 {
9997 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9998 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9999 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10000
10001 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10002 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10003 AssertRCReturn(rc, rc);
10004 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10005 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10006 0 /* GCPtrFaultAddress */);
10007 AssertRCReturn(rc, rc);
10008 }
10009 else
10010 {
10011 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10012 pVCpu->hm.s.u32HMError = uVector;
10013 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10014 }
10015 break;
10016 }
10017 }
10018 break;
10019 }
10020
10021 default:
10022 {
10023 pVCpu->hm.s.u32HMError = uExitIntInfo;
10024 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10025 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10026 break;
10027 }
10028 }
10029 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10030 return rc;
10031}
10032
10033
10034/**
10035 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10036 */
10037HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10038{
10039 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10040
10041 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
10042 hmR0VmxClearIntWindowExitVmcs(pVCpu);
10043
10044 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10045 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
10046 return VINF_SUCCESS;
10047}
10048
10049
10050/**
10051 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
10052 */
10053HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10054{
10055 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10056 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
10057 {
10058 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
10059 HMVMX_RETURN_UNEXPECTED_EXIT();
10060 }
10061
10062 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
10063
10064 /*
10065 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
10066 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
10067 */
10068 uint32_t uIntrState = 0;
10069 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10070 AssertRCReturn(rc, rc);
10071
10072 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
10073 if ( fBlockSti
10074 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
10075 {
10076 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10077 }
10078
10079 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
10080 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
10081
10082 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
10083 return VINF_SUCCESS;
10084}
10085
10086
10087/**
10088 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
10089 */
10090HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10091{
10092 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10093 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
10094 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10095}
10096
10097
10098/**
10099 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
10100 */
10101HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10102{
10103 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10104 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
10105 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10106}
10107
10108
10109/**
10110 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
10111 */
10112HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10113{
10114 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10115 PVM pVM = pVCpu->CTX_SUFF(pVM);
10116 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10117 if (RT_LIKELY(rc == VINF_SUCCESS))
10118 {
10119 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10120 Assert(pVmxTransient->cbInstr == 2);
10121 }
10122 else
10123 {
10124 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
10125 rc = VERR_EM_INTERPRETER;
10126 }
10127 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
10128 return rc;
10129}
10130
10131
10132/**
10133 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10134 */
10135HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10136{
10137 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10138 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10139 AssertRCReturn(rc, rc);
10140
10141 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10142 return VINF_EM_RAW_EMULATE_INSTR;
10143
10144 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10145 HMVMX_RETURN_UNEXPECTED_EXIT();
10146}
10147
10148
10149/**
10150 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10151 */
10152HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10153{
10154 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10155 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10156 AssertRCReturn(rc, rc);
10157
10158 PVM pVM = pVCpu->CTX_SUFF(pVM);
10159 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10160 if (RT_LIKELY(rc == VINF_SUCCESS))
10161 {
10162 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10163 Assert(pVmxTransient->cbInstr == 2);
10164 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10165 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10166 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10167 }
10168 else
10169 rc = VERR_EM_INTERPRETER;
10170 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10171 return rc;
10172}
10173
10174
10175/**
10176 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10177 */
10178HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10179{
10180 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10181 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10182 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10183 AssertRCReturn(rc, rc);
10184
10185 PVM pVM = pVCpu->CTX_SUFF(pVM);
10186 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10187 if (RT_LIKELY(rc == VINF_SUCCESS))
10188 {
10189 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10190 Assert(pVmxTransient->cbInstr == 3);
10191 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10192 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10193 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10194 }
10195 else
10196 {
10197 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10198 rc = VERR_EM_INTERPRETER;
10199 }
10200 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10201 return rc;
10202}
10203
10204
10205/**
10206 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10207 */
10208HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10209{
10210 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10211 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10212 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10213 AssertRCReturn(rc, rc);
10214
10215 PVM pVM = pVCpu->CTX_SUFF(pVM);
10216 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10217 if (RT_LIKELY(rc == VINF_SUCCESS))
10218 {
10219 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10220 Assert(pVmxTransient->cbInstr == 2);
10221 }
10222 else
10223 {
10224 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10225 rc = VERR_EM_INTERPRETER;
10226 }
10227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10228 return rc;
10229}
10230
10231
10232/**
10233 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10234 */
10235HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10236{
10237 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10238
10239 int rc = VERR_NOT_SUPPORTED;
10240 if (GIMAreHypercallsEnabled(pVCpu))
10241 {
10242 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10243 AssertRCReturn(rc, rc);
10244
10245 rc = GIMHypercall(pVCpu, pMixedCtx);
10246 }
10247 if (rc != VINF_SUCCESS)
10248 {
10249 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10250 rc = VINF_SUCCESS;
10251 }
10252
10253 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10254 return rc;
10255}
10256
10257
10258/**
10259 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10260 */
10261HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10262{
10263 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10264 PVM pVM = pVCpu->CTX_SUFF(pVM);
10265 Assert(!pVM->hm.s.fNestedPaging);
10266
10267 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10268 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10269 AssertRCReturn(rc, rc);
10270
10271 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10272 rc = VBOXSTRICTRC_VAL(rc2);
10273 if (RT_LIKELY(rc == VINF_SUCCESS))
10274 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10275 else
10276 {
10277 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10278 pVmxTransient->uExitQualification, rc));
10279 }
10280 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10281 return rc;
10282}
10283
10284
10285/**
10286 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10287 */
10288HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10289{
10290 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10291 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10292 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10293 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10294 AssertRCReturn(rc, rc);
10295
10296 PVM pVM = pVCpu->CTX_SUFF(pVM);
10297 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10298 if (RT_LIKELY(rc == VINF_SUCCESS))
10299 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10300 else
10301 {
10302 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10303 rc = VERR_EM_INTERPRETER;
10304 }
10305 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10306 return rc;
10307}
10308
10309
10310/**
10311 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10312 */
10313HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10314{
10315 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10316 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10317 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10318 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10319 AssertRCReturn(rc, rc);
10320
10321 PVM pVM = pVCpu->CTX_SUFF(pVM);
10322 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10323 rc = VBOXSTRICTRC_VAL(rc2);
10324 if (RT_LIKELY( rc == VINF_SUCCESS
10325 || rc == VINF_EM_HALT))
10326 {
10327 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10328 AssertRCReturn(rc3, rc3);
10329
10330 if ( rc == VINF_EM_HALT
10331 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10332 {
10333 rc = VINF_SUCCESS;
10334 }
10335 }
10336 else
10337 {
10338 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10339 rc = VERR_EM_INTERPRETER;
10340 }
10341 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10342 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10343 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10344 return rc;
10345}
10346
10347
10348/**
10349 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10350 */
10351HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10352{
10353 /*
10354 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10355 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10356 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10357 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10358 */
10359 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10360 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10361 HMVMX_RETURN_UNEXPECTED_EXIT();
10362}
10363
10364
10365/**
10366 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10367 */
10368HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10369{
10370 /*
10371 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10372 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
10373 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10374 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10375 */
10376 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10377 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10378 HMVMX_RETURN_UNEXPECTED_EXIT();
10379}
10380
10381
10382/**
10383 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10384 */
10385HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10386{
10387 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10388 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10389 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10390 HMVMX_RETURN_UNEXPECTED_EXIT();
10391}
10392
10393
10394/**
10395 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10396 */
10397HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10398{
10399 /*
10400 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10401 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10402 * See Intel spec. 25.3 "Other Causes of VM-exits".
10403 */
10404 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10405 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10406 HMVMX_RETURN_UNEXPECTED_EXIT();
10407}
10408
10409
10410/**
10411 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10412 * VM-exit.
10413 */
10414HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10415{
10416 /*
10417 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10418 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10419 *
10420 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10421 * See Intel spec. "23.8 Restrictions on VMX operation".
10422 */
10423 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10424 return VINF_SUCCESS;
10425}
10426
10427
10428/**
10429 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10430 * VM-exit.
10431 */
10432HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10433{
10434 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10435 return VINF_EM_RESET;
10436}
10437
10438
10439/**
10440 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10441 */
10442HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10443{
10444 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10445 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10446 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10447 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10448 AssertRCReturn(rc, rc);
10449
10450 pMixedCtx->rip++;
10451 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10452 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10453 rc = VINF_SUCCESS;
10454 else
10455 rc = VINF_EM_HALT;
10456
10457 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10458 return rc;
10459}
10460
10461
10462/**
10463 * VM-exit handler for instructions that result in a #UD exception delivered to
10464 * the guest.
10465 */
10466HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10467{
10468 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10469 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10470 return VINF_SUCCESS;
10471}
10472
10473
10474/**
10475 * VM-exit handler for expiry of the VMX preemption timer.
10476 */
10477HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10478{
10479 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10480
10481 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10482 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10483
10484 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10485 PVM pVM = pVCpu->CTX_SUFF(pVM);
10486 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10487 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10488 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10489}
10490
10491
10492/**
10493 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10494 */
10495HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10496{
10497 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10498
10499 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10500 /** @todo check if XSETBV is supported by the recompiler. */
10501 return VERR_EM_INTERPRETER;
10502}
10503
10504
10505/**
10506 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10507 */
10508HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10509{
10510 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10511
10512 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10513 /** @todo implement EMInterpretInvpcid() */
10514 return VERR_EM_INTERPRETER;
10515}
10516
10517
10518/**
10519 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10520 * Error VM-exit.
10521 */
10522HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10523{
10524 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10525 AssertRCReturn(rc, rc);
10526
10527 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10528 AssertRCReturn(rc, rc);
10529
10530 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10531 NOREF(uInvalidReason);
10532
10533#ifdef VBOX_STRICT
10534 uint32_t uIntrState;
10535 HMVMXHCUINTREG uHCReg;
10536 uint64_t u64Val;
10537 uint32_t u32Val;
10538
10539 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10540 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10541 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10542 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10543 AssertRCReturn(rc, rc);
10544
10545 Log4(("uInvalidReason %u\n", uInvalidReason));
10546 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10547 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10548 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10549 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10550
10551 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10552 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10553 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10554 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10555 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10556 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10557 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10558 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10559 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10560 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10561 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10562 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10563#else
10564 NOREF(pVmxTransient);
10565#endif
10566
10567 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10568 return VERR_VMX_INVALID_GUEST_STATE;
10569}
10570
10571
10572/**
10573 * VM-exit handler for VM-entry failure due to an MSR-load
10574 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10575 */
10576HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10577{
10578 NOREF(pVmxTransient);
10579 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10580 HMVMX_RETURN_UNEXPECTED_EXIT();
10581}
10582
10583
10584/**
10585 * VM-exit handler for VM-entry failure due to a machine-check event
10586 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10587 */
10588HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10589{
10590 NOREF(pVmxTransient);
10591 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10592 HMVMX_RETURN_UNEXPECTED_EXIT();
10593}
10594
10595
10596/**
10597 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10598 * theory.
10599 */
10600HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10601{
10602 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10603 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10604 return VERR_VMX_UNDEFINED_EXIT_CODE;
10605}
10606
10607
10608/**
10609 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10610 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10611 * Conditional VM-exit.
10612 */
10613HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10614{
10615 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10616
10617 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10618 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10619 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10620 return VERR_EM_INTERPRETER;
10621 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10622 HMVMX_RETURN_UNEXPECTED_EXIT();
10623}
10624
10625
10626/**
10627 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10628 */
10629HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10630{
10631 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10632
10633 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10634 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10635 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10636 return VERR_EM_INTERPRETER;
10637 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10638 HMVMX_RETURN_UNEXPECTED_EXIT();
10639}
10640
10641
10642/**
10643 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10644 */
10645HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10646{
10647 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10648
10649 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10650 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10651 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10652 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10653 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10654 {
10655 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10656 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10657 }
10658 AssertRCReturn(rc, rc);
10659 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10660
10661#ifdef VBOX_STRICT
10662 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10663 {
10664 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10665 && pMixedCtx->ecx != MSR_K6_EFER)
10666 {
10667 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10668 HMVMX_RETURN_UNEXPECTED_EXIT();
10669 }
10670# if HC_ARCH_BITS == 64
10671 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10672 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10673 {
10674 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10675 HMVMX_RETURN_UNEXPECTED_EXIT();
10676 }
10677# endif
10678 }
10679#endif
10680
10681 PVM pVM = pVCpu->CTX_SUFF(pVM);
10682 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10683 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10684 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10685 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10686 if (RT_LIKELY(rc == VINF_SUCCESS))
10687 {
10688 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10689 Assert(pVmxTransient->cbInstr == 2);
10690 }
10691 return rc;
10692}
10693
10694
10695/**
10696 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10697 */
10698HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10699{
10700 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10701 PVM pVM = pVCpu->CTX_SUFF(pVM);
10702 int rc = VINF_SUCCESS;
10703
10704 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10705 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10706 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10707 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10708 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10709 {
10710 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10711 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10712 }
10713 AssertRCReturn(rc, rc);
10714 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
10715
10716 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10717 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10718 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10719
10720 if (RT_LIKELY(rc == VINF_SUCCESS))
10721 {
10722 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10723
10724 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10725 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10726 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10727 {
10728 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10729 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10730 EMInterpretWrmsr() changes it. */
10731 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10732 }
10733 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10734 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10735 else if (pMixedCtx->ecx == MSR_K6_EFER)
10736 {
10737 /*
10738 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10739 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10740 * the other bits as well, SCE and NXE. See @bugref{7368}.
10741 */
10742 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10743 }
10744
10745 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10746 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10747 {
10748 switch (pMixedCtx->ecx)
10749 {
10750 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10751 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10752 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10753 case MSR_K8_FS_BASE: /* no break */
10754 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10755 case MSR_K6_EFER: /* already handled above */ break;
10756 default:
10757 {
10758 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10759 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10760#if HC_ARCH_BITS == 64
10761 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10762 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10763#endif
10764 break;
10765 }
10766 }
10767 }
10768#ifdef VBOX_STRICT
10769 else
10770 {
10771 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10772 switch (pMixedCtx->ecx)
10773 {
10774 case MSR_IA32_SYSENTER_CS:
10775 case MSR_IA32_SYSENTER_EIP:
10776 case MSR_IA32_SYSENTER_ESP:
10777 case MSR_K8_FS_BASE:
10778 case MSR_K8_GS_BASE:
10779 {
10780 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10781 HMVMX_RETURN_UNEXPECTED_EXIT();
10782 }
10783
10784 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10785 default:
10786 {
10787 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10788 {
10789 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10790 if (pMixedCtx->ecx != MSR_K6_EFER)
10791 {
10792 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10793 pMixedCtx->ecx));
10794 HMVMX_RETURN_UNEXPECTED_EXIT();
10795 }
10796 }
10797
10798#if HC_ARCH_BITS == 64
10799 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10800 {
10801 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10802 HMVMX_RETURN_UNEXPECTED_EXIT();
10803 }
10804#endif
10805 break;
10806 }
10807 }
10808 }
10809#endif /* VBOX_STRICT */
10810 }
10811 return rc;
10812}
10813
10814
10815/**
10816 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10817 */
10818HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10819{
10820 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10821
10822 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10823 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10824 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10825 return VERR_EM_INTERPRETER;
10826 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10827 HMVMX_RETURN_UNEXPECTED_EXIT();
10828}
10829
10830
10831/**
10832 * VM-exit handler for when the TPR value is lowered below the specified
10833 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10834 */
10835HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10836{
10837 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10838 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10839
10840 /*
10841 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10842 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10843 * resume guest execution.
10844 */
10845 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10846 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10847 return VINF_SUCCESS;
10848}
10849
10850
10851/**
10852 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10853 * VM-exit.
10854 *
10855 * @retval VINF_SUCCESS when guest execution can continue.
10856 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10857 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10858 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10859 * recompiler.
10860 */
10861HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10862{
10863 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10864 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10865 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10866 AssertRCReturn(rc, rc);
10867
10868 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10869 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10870 PVM pVM = pVCpu->CTX_SUFF(pVM);
10871 switch (uAccessType)
10872 {
10873 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10874 {
10875#if 0
10876 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10877 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10878#else
10879 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10880 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10881 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10882#endif
10883 AssertRCReturn(rc, rc);
10884
10885 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10886 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10887 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10888 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10889
10890 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10891 {
10892 case 0: /* CR0 */
10893 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10894 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10895 break;
10896 case 2: /* CR2 */
10897 /* Nothing to do here, CR2 it's not part of the VMCS. */
10898 break;
10899 case 3: /* CR3 */
10900 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10901 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10902 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10903 break;
10904 case 4: /* CR4 */
10905 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10906 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10907 break;
10908 case 8: /* CR8 */
10909 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10910 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10911 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10912 break;
10913 default:
10914 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10915 break;
10916 }
10917
10918 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10919 break;
10920 }
10921
10922 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10923 {
10924 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10925 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10926 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10927 AssertRCReturn(rc, rc);
10928 Assert( !pVM->hm.s.fNestedPaging
10929 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10930 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10931
10932 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10933 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10934 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10935
10936 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10937 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10938 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10939 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10940 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10941 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10942 break;
10943 }
10944
10945 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10946 {
10947 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10948 AssertRCReturn(rc, rc);
10949 rc = EMInterpretCLTS(pVM, pVCpu);
10950 AssertRCReturn(rc, rc);
10951 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10952 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10953 Log4(("CRX CLTS write rc=%d\n", rc));
10954 break;
10955 }
10956
10957 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10958 {
10959 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10960 AssertRCReturn(rc, rc);
10961 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10962 if (RT_LIKELY(rc == VINF_SUCCESS))
10963 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10964 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10965 Log4(("CRX LMSW write rc=%d\n", rc));
10966 break;
10967 }
10968
10969 default:
10970 {
10971 AssertMsgFailed(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType));
10972 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10973 }
10974 }
10975
10976 /* Validate possible error codes. */
10977 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10978 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10979 if (RT_SUCCESS(rc))
10980 {
10981 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10982 AssertRCReturn(rc2, rc2);
10983 }
10984
10985 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10986 return rc;
10987}
10988
10989
10990/**
10991 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10992 * VM-exit.
10993 */
10994HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10995{
10996 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10997 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10998
10999 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11000 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11001 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11002 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
11003 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
11004 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
11005 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
11006 AssertRCReturn(rc2, rc2);
11007
11008 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
11009 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
11010 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
11011 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
11012 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
11013 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
11014 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11015 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
11016
11017 /* I/O operation lookup arrays. */
11018 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
11019 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
11020
11021 VBOXSTRICTRC rcStrict;
11022 uint32_t const cbValue = s_aIOSizes[uIOWidth];
11023 uint32_t const cbInstr = pVmxTransient->cbInstr;
11024 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
11025 PVM pVM = pVCpu->CTX_SUFF(pVM);
11026 if (fIOString)
11027 {
11028#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
11029 /*
11030 * INS/OUTS - I/O String instruction.
11031 *
11032 * Use instruction-information if available, otherwise fall back on
11033 * interpreting the instruction.
11034 */
11035 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11036 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
11037 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
11038 {
11039 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11040 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11041 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11042 AssertRCReturn(rc2, rc2);
11043 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
11044 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
11045 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
11046 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
11047 if (fIOWrite)
11048 {
11049 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
11050 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
11051 }
11052 else
11053 {
11054 /*
11055 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
11056 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
11057 * See Intel Instruction spec. for "INS".
11058 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
11059 */
11060 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
11061 }
11062 }
11063 else
11064 {
11065 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
11066 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11067 AssertRCReturn(rc2, rc2);
11068 rcStrict = IEMExecOne(pVCpu);
11069 }
11070 /** @todo IEM needs to be setting these flags somehow. */
11071 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11072 fUpdateRipAlready = true;
11073#else
11074 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11075 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
11076 if (RT_SUCCESS(rcStrict))
11077 {
11078 if (fIOWrite)
11079 {
11080 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11081 (DISCPUMODE)pDis->uAddrMode, cbValue);
11082 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
11083 }
11084 else
11085 {
11086 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
11087 (DISCPUMODE)pDis->uAddrMode, cbValue);
11088 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
11089 }
11090 }
11091 else
11092 {
11093 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
11094 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
11095 }
11096#endif
11097 }
11098 else
11099 {
11100 /*
11101 * IN/OUT - I/O instruction.
11102 */
11103 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
11104 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
11105 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
11106 if (fIOWrite)
11107 {
11108 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
11109 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11110 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11111 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
11112 }
11113 else
11114 {
11115 uint32_t u32Result = 0;
11116 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
11117 if (IOM_SUCCESS(rcStrict))
11118 {
11119 /* Save result of I/O IN instr. in AL/AX/EAX. */
11120 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
11121 }
11122 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11123 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
11124 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
11125 }
11126 }
11127
11128 if (IOM_SUCCESS(rcStrict))
11129 {
11130 if (!fUpdateRipAlready)
11131 {
11132 pMixedCtx->rip += cbInstr;
11133 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11134 }
11135
11136 /*
11137 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11138 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11139 */
11140 if (fIOString)
11141 {
11142 /** @todo Single-step for INS/OUTS with REP prefix? */
11143 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11144 }
11145 else if (fStepping)
11146 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11147
11148 /*
11149 * If any I/O breakpoints are armed, we need to check if one triggered
11150 * and take appropriate action.
11151 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11152 */
11153 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11154 AssertRCReturn(rc2, rc2);
11155
11156 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11157 * execution engines about whether hyper BPs and such are pending. */
11158 uint32_t const uDr7 = pMixedCtx->dr[7];
11159 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11160 && X86_DR7_ANY_RW_IO(uDr7)
11161 && (pMixedCtx->cr4 & X86_CR4_DE))
11162 || DBGFBpIsHwIoArmed(pVM)))
11163 {
11164 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11165
11166 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11167 VMMRZCallRing3Disable(pVCpu);
11168 HM_DISABLE_PREEMPT_IF_NEEDED();
11169
11170 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
11171
11172 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11173 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11174 {
11175 /* Raise #DB. */
11176 if (fIsGuestDbgActive)
11177 ASMSetDR6(pMixedCtx->dr[6]);
11178 if (pMixedCtx->dr[7] != uDr7)
11179 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11180
11181 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11182 }
11183 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11184 else if ( rcStrict2 != VINF_SUCCESS
11185 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11186 rcStrict = rcStrict2;
11187
11188 HM_RESTORE_PREEMPT_IF_NEEDED();
11189 VMMRZCallRing3Enable(pVCpu);
11190 }
11191 }
11192
11193#ifdef DEBUG
11194 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11195 Assert(!fIOWrite);
11196 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11197 Assert(fIOWrite);
11198 else
11199 {
11200 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11201 * statuses, that the VMM device and some others may return. See
11202 * IOM_SUCCESS() for guidance. */
11203 AssertMsg( RT_FAILURE(rcStrict)
11204 || rcStrict == VINF_SUCCESS
11205 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11206 || rcStrict == VINF_EM_DBG_BREAKPOINT
11207 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11208 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11209 }
11210#endif
11211
11212 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11213 return VBOXSTRICTRC_TODO(rcStrict);
11214}
11215
11216
11217/**
11218 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11219 * VM-exit.
11220 */
11221HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11222{
11223 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11224
11225 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11226 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11227 AssertRCReturn(rc, rc);
11228 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11229 {
11230 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11231 AssertRCReturn(rc, rc);
11232 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11233 {
11234 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11235
11236 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11237 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11238
11239 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11240 Assert(!pVCpu->hm.s.Event.fPending);
11241 pVCpu->hm.s.Event.fPending = true;
11242 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11243 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11244 AssertRCReturn(rc, rc);
11245 if (fErrorCodeValid)
11246 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11247 else
11248 pVCpu->hm.s.Event.u32ErrCode = 0;
11249 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11250 && uVector == X86_XCPT_PF)
11251 {
11252 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11253 }
11254
11255 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11256 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11257 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11258 }
11259 }
11260
11261 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11262 * emulation. */
11263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11264 return VERR_EM_INTERPRETER;
11265}
11266
11267
11268/**
11269 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11270 */
11271HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11272{
11273 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11274 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11275 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11276 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11277 AssertRCReturn(rc, rc);
11278 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11279 return VINF_EM_DBG_STEPPED;
11280}
11281
11282
11283/**
11284 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11285 */
11286HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11287{
11288 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11289
11290 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11291 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11292 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11293 {
11294 if (rc == VINF_HM_DOUBLE_FAULT)
11295 rc = VINF_SUCCESS;
11296 return rc;
11297 }
11298
11299#if 0
11300 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11301 * just sync the whole thing. */
11302 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11303#else
11304 /* Aggressive state sync. for now. */
11305 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11306 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11307 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11308#endif
11309 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11310 AssertRCReturn(rc, rc);
11311
11312 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11313 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11314 switch (uAccessType)
11315 {
11316 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11317 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11318 {
11319 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11320 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
11321 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11322
11323 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11324 GCPhys &= PAGE_BASE_GC_MASK;
11325 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11326 PVM pVM = pVCpu->CTX_SUFF(pVM);
11327 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11328 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11329
11330 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11331 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
11332 CPUMCTX2CORE(pMixedCtx), GCPhys);
11333 rc = VBOXSTRICTRC_VAL(rc2);
11334 Log4(("ApicAccess rc=%d\n", rc));
11335 if ( rc == VINF_SUCCESS
11336 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11337 || rc == VERR_PAGE_NOT_PRESENT)
11338 {
11339 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11340 | HM_CHANGED_GUEST_RSP
11341 | HM_CHANGED_GUEST_RFLAGS
11342 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11343 rc = VINF_SUCCESS;
11344 }
11345 break;
11346 }
11347
11348 default:
11349 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11350 rc = VINF_EM_RAW_EMULATE_INSTR;
11351 break;
11352 }
11353
11354 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11355 if (rc != VINF_SUCCESS)
11356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccessToR3);
11357 return rc;
11358}
11359
11360
11361/**
11362 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11363 * VM-exit.
11364 */
11365HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11366{
11367 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11368
11369 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11370 if (pVmxTransient->fWasGuestDebugStateActive)
11371 {
11372 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11373 HMVMX_RETURN_UNEXPECTED_EXIT();
11374 }
11375
11376 int rc = VERR_INTERNAL_ERROR_5;
11377 if ( !DBGFIsStepping(pVCpu)
11378 && !pVCpu->hm.s.fSingleInstruction
11379 && !pVmxTransient->fWasHyperDebugStateActive)
11380 {
11381 /* Don't intercept MOV DRx and #DB any more. */
11382 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11383 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11384 AssertRCReturn(rc, rc);
11385
11386 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11387 {
11388#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11389 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11390 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11391 AssertRCReturn(rc, rc);
11392#endif
11393 }
11394
11395 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11396 VMMRZCallRing3Disable(pVCpu);
11397 HM_DISABLE_PREEMPT_IF_NEEDED();
11398
11399 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11400 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11401 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11402
11403 HM_RESTORE_PREEMPT_IF_NEEDED();
11404 VMMRZCallRing3Enable(pVCpu);
11405
11406#ifdef VBOX_WITH_STATISTICS
11407 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11408 AssertRCReturn(rc, rc);
11409 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11410 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11411 else
11412 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11413#endif
11414 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11415 return VINF_SUCCESS;
11416 }
11417
11418 /*
11419 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11420 * Update the segment registers and DR7 from the CPU.
11421 */
11422 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11423 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11424 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11425 AssertRCReturn(rc, rc);
11426 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11427
11428 PVM pVM = pVCpu->CTX_SUFF(pVM);
11429 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11430 {
11431 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11432 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11433 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11434 if (RT_SUCCESS(rc))
11435 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11436 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11437 }
11438 else
11439 {
11440 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11441 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11442 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11443 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11444 }
11445
11446 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11447 if (RT_SUCCESS(rc))
11448 {
11449 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11450 AssertRCReturn(rc2, rc2);
11451 }
11452 return rc;
11453}
11454
11455
11456/**
11457 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11458 * Conditional VM-exit.
11459 */
11460HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11461{
11462 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11463 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11464
11465 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11466 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11467 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11468 {
11469 if (rc == VINF_HM_DOUBLE_FAULT)
11470 rc = VINF_SUCCESS;
11471 return rc;
11472 }
11473
11474 RTGCPHYS GCPhys = 0;
11475 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11476
11477#if 0
11478 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11479#else
11480 /* Aggressive state sync. for now. */
11481 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11482 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11483 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11484#endif
11485 AssertRCReturn(rc, rc);
11486
11487 /*
11488 * If we succeed, resume guest execution.
11489 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11490 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11491 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11492 * weird case. See @bugref{6043}.
11493 */
11494 PVM pVM = pVCpu->CTX_SUFF(pVM);
11495 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11496 rc = VBOXSTRICTRC_VAL(rc2);
11497 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11498 if ( rc == VINF_SUCCESS
11499 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11500 || rc == VERR_PAGE_NOT_PRESENT)
11501 {
11502 /* Successfully handled MMIO operation. */
11503 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11504 | HM_CHANGED_GUEST_RSP
11505 | HM_CHANGED_GUEST_RFLAGS
11506 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11507 rc = VINF_SUCCESS;
11508 }
11509 return rc;
11510}
11511
11512
11513/**
11514 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11515 * VM-exit.
11516 */
11517HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11518{
11519 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11520 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11521
11522 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11523 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11524 if (RT_UNLIKELY(rc != VINF_SUCCESS))
11525 {
11526 if (rc == VINF_HM_DOUBLE_FAULT)
11527 rc = VINF_SUCCESS;
11528 return rc;
11529 }
11530
11531 RTGCPHYS GCPhys = 0;
11532 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11533 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11534#if 0
11535 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11536#else
11537 /* Aggressive state sync. for now. */
11538 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11539 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11540 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11541#endif
11542 AssertRCReturn(rc, rc);
11543
11544 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11545 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11546
11547 RTGCUINT uErrorCode = 0;
11548 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11549 uErrorCode |= X86_TRAP_PF_ID;
11550 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11551 uErrorCode |= X86_TRAP_PF_RW;
11552 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11553 uErrorCode |= X86_TRAP_PF_P;
11554
11555 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11556
11557 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
11558 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11559
11560 /* Handle the pagefault trap for the nested shadow table. */
11561 PVM pVM = pVCpu->CTX_SUFF(pVM);
11562 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11563 TRPMResetTrap(pVCpu);
11564
11565 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11566 if ( rc == VINF_SUCCESS
11567 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11568 || rc == VERR_PAGE_NOT_PRESENT)
11569 {
11570 /* Successfully synced our nested page tables. */
11571 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11572 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11573 | HM_CHANGED_GUEST_RSP
11574 | HM_CHANGED_GUEST_RFLAGS);
11575 return VINF_SUCCESS;
11576 }
11577
11578 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11579 return rc;
11580}
11581
11582/** @} */
11583
11584/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11585/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11586/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11587
11588/** @name VM-exit exception handlers.
11589 * @{
11590 */
11591
11592/**
11593 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11594 */
11595static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11596{
11597 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11598 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11599
11600 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11601 AssertRCReturn(rc, rc);
11602
11603 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11604 {
11605 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11606 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11607
11608 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11609 * provides VM-exit instruction length. If this causes problem later,
11610 * disassemble the instruction like it's done on AMD-V. */
11611 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11612 AssertRCReturn(rc2, rc2);
11613 return rc;
11614 }
11615
11616 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11617 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11618 return rc;
11619}
11620
11621
11622/**
11623 * VM-exit exception handler for #BP (Breakpoint exception).
11624 */
11625static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11626{
11627 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11628 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11629
11630 /** @todo Try optimize this by not saving the entire guest state unless
11631 * really needed. */
11632 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11633 AssertRCReturn(rc, rc);
11634
11635 PVM pVM = pVCpu->CTX_SUFF(pVM);
11636 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11637 if (rc == VINF_EM_RAW_GUEST_TRAP)
11638 {
11639 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11640 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11641 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11642 AssertRCReturn(rc, rc);
11643
11644 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11645 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11646 }
11647
11648 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11649 return rc;
11650}
11651
11652
11653/**
11654 * VM-exit exception handler for #DB (Debug exception).
11655 */
11656static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11657{
11658 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11659 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11660 Log6(("XcptDB\n"));
11661
11662 /*
11663 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
11664 * for processing.
11665 */
11666 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11667 AssertRCReturn(rc, rc);
11668
11669 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11670 uint64_t uDR6 = X86_DR6_INIT_VAL;
11671 uDR6 |= ( pVmxTransient->uExitQualification
11672 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11673
11674 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11675 if (rc == VINF_EM_RAW_GUEST_TRAP)
11676 {
11677 /*
11678 * The exception was for the guest. Update DR6, DR7.GD and
11679 * IA32_DEBUGCTL.LBR before forwarding it.
11680 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11681 */
11682 VMMRZCallRing3Disable(pVCpu);
11683 HM_DISABLE_PREEMPT_IF_NEEDED();
11684
11685 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11686 pMixedCtx->dr[6] |= uDR6;
11687 if (CPUMIsGuestDebugStateActive(pVCpu))
11688 ASMSetDR6(pMixedCtx->dr[6]);
11689
11690 HM_RESTORE_PREEMPT_IF_NEEDED();
11691 VMMRZCallRing3Enable(pVCpu);
11692
11693 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11694 AssertRCReturn(rc, rc);
11695
11696 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11697 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11698
11699 /* Paranoia. */
11700 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11701 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11702
11703 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11704 AssertRCReturn(rc, rc);
11705
11706 /*
11707 * Raise #DB in the guest.
11708 *
11709 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11710 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11711 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11712 *
11713 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11714 */
11715 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11716 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11717 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11718 AssertRCReturn(rc, rc);
11719 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11720 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11721 return VINF_SUCCESS;
11722 }
11723
11724 /*
11725 * Not a guest trap, must be a hypervisor related debug event then.
11726 * Update DR6 in case someone is interested in it.
11727 */
11728 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11729 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11730 CPUMSetHyperDR6(pVCpu, uDR6);
11731
11732 return rc;
11733}
11734
11735
11736/**
11737 * VM-exit exception handler for #NM (Device-not-available exception: floating
11738 * point exception).
11739 */
11740static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11741{
11742 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11743
11744 /* We require CR0 and EFER. EFER is always up-to-date. */
11745 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11746 AssertRCReturn(rc, rc);
11747
11748 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11749 VMMRZCallRing3Disable(pVCpu);
11750 HM_DISABLE_PREEMPT_IF_NEEDED();
11751
11752 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11753 if (pVmxTransient->fWasGuestFPUStateActive)
11754 {
11755 rc = VINF_EM_RAW_GUEST_TRAP;
11756 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11757 }
11758 else
11759 {
11760#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11761 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11762#endif
11763 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11764 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11765 }
11766
11767 HM_RESTORE_PREEMPT_IF_NEEDED();
11768 VMMRZCallRing3Enable(pVCpu);
11769
11770 if (rc == VINF_SUCCESS)
11771 {
11772 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11773 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11774 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11775 pVCpu->hm.s.fUseGuestFpu = true;
11776 }
11777 else
11778 {
11779 /* Forward #NM to the guest. */
11780 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11781 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11782 AssertRCReturn(rc, rc);
11783 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11784 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11785 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11786 }
11787
11788 return VINF_SUCCESS;
11789}
11790
11791
11792/**
11793 * VM-exit exception handler for #GP (General-protection exception).
11794 *
11795 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11796 */
11797static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11798{
11799 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11800 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11801
11802 int rc = VERR_INTERNAL_ERROR_5;
11803 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11804 {
11805#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11806 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11807 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11808 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11809 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11810 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11811 AssertRCReturn(rc, rc);
11812 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11813 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11814 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11815 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11816 return rc;
11817#else
11818 /* We don't intercept #GP. */
11819 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11820 NOREF(pVmxTransient);
11821 return VERR_VMX_UNEXPECTED_EXCEPTION;
11822#endif
11823 }
11824
11825 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11826 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11827
11828 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11829 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11830 AssertRCReturn(rc, rc);
11831
11832 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11833 uint32_t cbOp = 0;
11834 PVM pVM = pVCpu->CTX_SUFF(pVM);
11835 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11836 if (RT_SUCCESS(rc))
11837 {
11838 rc = VINF_SUCCESS;
11839 Assert(cbOp == pDis->cbInstr);
11840 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11841 switch (pDis->pCurInstr->uOpcode)
11842 {
11843 case OP_CLI:
11844 {
11845 pMixedCtx->eflags.Bits.u1IF = 0;
11846 pMixedCtx->eflags.Bits.u1RF = 0;
11847 pMixedCtx->rip += pDis->cbInstr;
11848 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11849 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11850 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11851 break;
11852 }
11853
11854 case OP_STI:
11855 {
11856 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
11857 pMixedCtx->eflags.Bits.u1IF = 1;
11858 pMixedCtx->eflags.Bits.u1RF = 0;
11859 pMixedCtx->rip += pDis->cbInstr;
11860 if (!fOldIF)
11861 {
11862 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11863 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11864 }
11865 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11866 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11867 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11868 break;
11869 }
11870
11871 case OP_HLT:
11872 {
11873 rc = VINF_EM_HALT;
11874 pMixedCtx->rip += pDis->cbInstr;
11875 pMixedCtx->eflags.Bits.u1RF = 0;
11876 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11877 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11878 break;
11879 }
11880
11881 case OP_POPF:
11882 {
11883 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11884 uint32_t cbParm;
11885 uint32_t uMask;
11886 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11887 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11888 {
11889 cbParm = 4;
11890 uMask = 0xffffffff;
11891 }
11892 else
11893 {
11894 cbParm = 2;
11895 uMask = 0xffff;
11896 }
11897
11898 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11899 RTGCPTR GCPtrStack = 0;
11900 X86EFLAGS Eflags;
11901 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11902 &GCPtrStack);
11903 if (RT_SUCCESS(rc))
11904 {
11905 Assert(sizeof(Eflags.u32) >= cbParm);
11906 Eflags.u32 = 0;
11907 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11908 }
11909 if (RT_FAILURE(rc))
11910 {
11911 rc = VERR_EM_INTERPRETER;
11912 break;
11913 }
11914 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11915 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
11916 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11917 pMixedCtx->esp += cbParm;
11918 pMixedCtx->esp &= uMask;
11919 pMixedCtx->rip += pDis->cbInstr;
11920 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11921 | HM_CHANGED_GUEST_RSP
11922 | HM_CHANGED_GUEST_RFLAGS);
11923 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11924 if (fStepping)
11925 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11926
11927 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11928 break;
11929 }
11930
11931 case OP_PUSHF:
11932 {
11933 uint32_t cbParm;
11934 uint32_t uMask;
11935 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11936 {
11937 cbParm = 4;
11938 uMask = 0xffffffff;
11939 }
11940 else
11941 {
11942 cbParm = 2;
11943 uMask = 0xffff;
11944 }
11945
11946 /* Get the stack pointer & push the contents of eflags onto the stack. */
11947 RTGCPTR GCPtrStack = 0;
11948 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11949 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11950 if (RT_FAILURE(rc))
11951 {
11952 rc = VERR_EM_INTERPRETER;
11953 break;
11954 }
11955 X86EFLAGS Eflags = pMixedCtx->eflags;
11956 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11957 Eflags.Bits.u1RF = 0;
11958 Eflags.Bits.u1VM = 0;
11959
11960 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11961 if (RT_FAILURE(rc))
11962 {
11963 rc = VERR_EM_INTERPRETER;
11964 break;
11965 }
11966 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11967 pMixedCtx->esp -= cbParm;
11968 pMixedCtx->esp &= uMask;
11969 pMixedCtx->rip += pDis->cbInstr;
11970 pMixedCtx->eflags.Bits.u1RF = 0;
11971 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11972 | HM_CHANGED_GUEST_RSP
11973 | HM_CHANGED_GUEST_RFLAGS);
11974 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11975 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11976 break;
11977 }
11978
11979 case OP_IRET:
11980 {
11981 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11982 * instruction reference. */
11983 RTGCPTR GCPtrStack = 0;
11984 uint32_t uMask = 0xffff;
11985 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11986 uint16_t aIretFrame[3];
11987 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11988 {
11989 rc = VERR_EM_INTERPRETER;
11990 break;
11991 }
11992 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11993 &GCPtrStack);
11994 if (RT_SUCCESS(rc))
11995 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11996 if (RT_FAILURE(rc))
11997 {
11998 rc = VERR_EM_INTERPRETER;
11999 break;
12000 }
12001 pMixedCtx->eip = 0;
12002 pMixedCtx->ip = aIretFrame[0];
12003 pMixedCtx->cs.Sel = aIretFrame[1];
12004 pMixedCtx->cs.ValidSel = aIretFrame[1];
12005 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
12006 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
12007 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
12008 pMixedCtx->sp += sizeof(aIretFrame);
12009 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12010 | HM_CHANGED_GUEST_SEGMENT_REGS
12011 | HM_CHANGED_GUEST_RSP
12012 | HM_CHANGED_GUEST_RFLAGS);
12013 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
12014 if (fStepping)
12015 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
12016 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
12017 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
12018 break;
12019 }
12020
12021 case OP_INT:
12022 {
12023 uint16_t uVector = pDis->Param1.uValue & 0xff;
12024 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
12025 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12026 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12027 break;
12028 }
12029
12030 case OP_INTO:
12031 {
12032 if (pMixedCtx->eflags.Bits.u1OF)
12033 {
12034 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
12035 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
12036 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
12037 }
12038 else
12039 {
12040 pMixedCtx->eflags.Bits.u1RF = 0;
12041 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12042 }
12043 break;
12044 }
12045
12046 default:
12047 {
12048 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
12049 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
12050 EMCODETYPE_SUPERVISOR);
12051 rc = VBOXSTRICTRC_VAL(rc2);
12052 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
12053 /** @todo We have to set pending-debug exceptions here when the guest is
12054 * single-stepping depending on the instruction that was interpreted. */
12055 Log4(("#GP rc=%Rrc\n", rc));
12056 break;
12057 }
12058 }
12059 }
12060 else
12061 rc = VERR_EM_INTERPRETER;
12062
12063 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
12064 ("#GP Unexpected rc=%Rrc\n", rc));
12065 return rc;
12066}
12067
12068
12069#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12070/**
12071 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
12072 * the exception reported in the VMX transient structure back into the VM.
12073 *
12074 * @remarks Requires uExitIntInfo in the VMX transient structure to be
12075 * up-to-date.
12076 */
12077static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12078{
12079 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12080
12081 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
12082 hmR0VmxCheckExitDueToEventDelivery(). */
12083 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12084 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12085 AssertRCReturn(rc, rc);
12086 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12087
12088#ifdef DEBUG_ramshankar
12089 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12090 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12091 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
12092#endif
12093
12094 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12095 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12096 return VINF_SUCCESS;
12097}
12098#endif
12099
12100
12101/**
12102 * VM-exit exception handler for #PF (Page-fault exception).
12103 */
12104static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12105{
12106 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12107 PVM pVM = pVCpu->CTX_SUFF(pVM);
12108 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12109 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12110 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12111 AssertRCReturn(rc, rc);
12112
12113#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
12114 if (pVM->hm.s.fNestedPaging)
12115 {
12116 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
12117 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
12118 {
12119 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12120 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12121 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
12122 }
12123 else
12124 {
12125 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12126 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12127 Log4(("Pending #DF due to vectoring #PF. NP\n"));
12128 }
12129 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12130 return rc;
12131 }
12132#else
12133 Assert(!pVM->hm.s.fNestedPaging);
12134 NOREF(pVM);
12135#endif
12136
12137 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
12138 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
12139 if (pVmxTransient->fVectoringPF)
12140 {
12141 Assert(pVCpu->hm.s.Event.fPending);
12142 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12143 }
12144
12145 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12146 AssertRCReturn(rc, rc);
12147
12148 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
12149 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
12150
12151 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
12152 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
12153 (RTGCPTR)pVmxTransient->uExitQualification);
12154
12155 Log4(("#PF: rc=%Rrc\n", rc));
12156 if (rc == VINF_SUCCESS)
12157 {
12158 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12159 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12160 * memory? We don't update the whole state here... */
12161 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12162 | HM_CHANGED_GUEST_RSP
12163 | HM_CHANGED_GUEST_RFLAGS
12164 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12165 TRPMResetTrap(pVCpu);
12166 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12167 return rc;
12168 }
12169
12170 if (rc == VINF_EM_RAW_GUEST_TRAP)
12171 {
12172 if (!pVmxTransient->fVectoringDoublePF)
12173 {
12174 /* It's a guest page fault and needs to be reflected to the guest. */
12175 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12176 TRPMResetTrap(pVCpu);
12177 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12178 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12179 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12180 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12181 }
12182 else
12183 {
12184 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12185 TRPMResetTrap(pVCpu);
12186 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12187 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12188 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12189 }
12190
12191 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12192 return VINF_SUCCESS;
12193 }
12194
12195 TRPMResetTrap(pVCpu);
12196 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12197 return rc;
12198}
12199
12200/** @} */
12201
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette